--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Collection;
+import java.util.Map;
+import java.util.Random;
+import java.io.IOException;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.analysis.KeywordAnalyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.SnapshotDeletionPolicy;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.ThreadInterruptedException;
+import org.junit.Test;
+
+//
+// This was developed for Lucene In Action,
+// http://lucenebook.com
+//
+
+public class TestSnapshotDeletionPolicy extends LuceneTestCase {
+ public static final String INDEX_PATH = "test.snapshots";
+
+ protected IndexWriterConfig getConfig(Random random, IndexDeletionPolicy dp) {
+ IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ if (dp != null) {
+ conf.setIndexDeletionPolicy(dp);
+ }
+ return conf;
+ }
+
+ protected void checkSnapshotExists(Directory dir, IndexCommit c) throws Exception {
+ String segFileName = c.getSegmentsFileName();
+ assertTrue("segments file not found in directory: " + segFileName, dir.fileExists(segFileName));
+ }
+
+ protected void checkMaxDoc(IndexCommit commit, int expectedMaxDoc) throws Exception {
+ IndexReader reader = IndexReader.open(commit, true);
+ try {
+ assertEquals(expectedMaxDoc, reader.maxDoc());
+ } finally {
+ reader.close();
+ }
+ }
+
+ protected void prepareIndexAndSnapshots(SnapshotDeletionPolicy sdp,
+ IndexWriter writer, int numSnapshots, String snapshotPrefix)
+ throws RuntimeException, IOException {
+ for (int i = 0; i < numSnapshots; i++) {
+ // create dummy document to trigger commit.
+ writer.addDocument(new Document());
+ writer.commit();
+ sdp.snapshot(snapshotPrefix + i);
+ }
+ }
+
+ protected SnapshotDeletionPolicy getDeletionPolicy() throws IOException {
+ return getDeletionPolicy(null);
+ }
+
+ protected SnapshotDeletionPolicy getDeletionPolicy(Map<String, String> snapshots) throws IOException {
+ return new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), snapshots);
+ }
+
+ protected void assertSnapshotExists(Directory dir, SnapshotDeletionPolicy sdp, int numSnapshots) throws Exception {
+ for (int i = 0; i < numSnapshots; i++) {
+ IndexCommit snapshot = sdp.getSnapshot("snapshot" + i);
+ checkMaxDoc(snapshot, i + 1);
+ checkSnapshotExists(dir, snapshot);
+ }
+ }
+
+ @Test
+ public void testSnapshotDeletionPolicy() throws Exception {
+ Directory fsDir = newDirectory();
+ runTest(random, fsDir);
+ fsDir.close();
+ }
+
+ private void runTest(Random random, Directory dir) throws Exception {
+ // Run for ~1 seconds
+ final long stopTime = System.currentTimeMillis() + 1000;
+
+ SnapshotDeletionPolicy dp = getDeletionPolicy();
+ final IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(dp)
+ .setMaxBufferedDocs(2));
+ writer.commit();
+
+ final Thread t = new Thread() {
+ @Override
+ public void run() {
+ Document doc = new Document();
+ doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ do {
+ for(int i=0;i<27;i++) {
+ try {
+ writer.addDocument(doc);
+ } catch (Throwable t) {
+ t.printStackTrace(System.out);
+ fail("addDocument failed");
+ }
+ if (i%2 == 0) {
+ try {
+ writer.commit();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ try {
+ Thread.sleep(1);
+ } catch (InterruptedException ie) {
+ throw new ThreadInterruptedException(ie);
+ }
+ } while(System.currentTimeMillis() < stopTime);
+ }
+ };
+
+ t.start();
+
+ // While the above indexing thread is running, take many
+ // backups:
+ do {
+ backupIndex(dir, dp);
+ Thread.sleep(20);
+ } while(t.isAlive());
+
+ t.join();
+
+ // Add one more document to force writer to commit a
+ // final segment, so deletion policy has a chance to
+ // delete again:
+ Document doc = new Document();
+ doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+ writer.addDocument(doc);
+
+ // Make sure we don't have any leftover files in the
+ // directory:
+ writer.close();
+ TestIndexWriter.assertNoUnreferencedFiles(dir, "some files were not deleted but should have been");
+ }
+
+ /**
+ * Example showing how to use the SnapshotDeletionPolicy to take a backup.
+ * This method does not really do a backup; instead, it reads every byte of
+ * every file just to test that the files indeed exist and are readable even
+ * while the index is changing.
+ */
+ public void backupIndex(Directory dir, SnapshotDeletionPolicy dp) throws Exception {
+ // To backup an index we first take a snapshot:
+ try {
+ copyFiles(dir, dp.snapshot("id"));
+ } finally {
+ // Make sure to release the snapshot, otherwise these
+ // files will never be deleted during this IndexWriter
+ // session:
+ dp.release("id");
+ }
+ }
+
+ private void copyFiles(Directory dir, IndexCommit cp) throws Exception {
+
+ // While we hold the snapshot, and nomatter how long
+ // we take to do the backup, the IndexWriter will
+ // never delete the files in the snapshot:
+ Collection<String> files = cp.getFileNames();
+ for (final String fileName : files) {
+ // NOTE: in a real backup you would not use
+ // readFile; you would need to use something else
+ // that copies the file to a backup location. This
+ // could even be a spawned shell process (eg "tar",
+ // "zip") that takes the list of files and builds a
+ // backup.
+ readFile(dir, fileName);
+ }
+ }
+
+ byte[] buffer = new byte[4096];
+
+ private void readFile(Directory dir, String name) throws Exception {
+ IndexInput input = dir.openInput(name);
+ try {
+ long size = dir.fileLength(name);
+ long bytesLeft = size;
+ while (bytesLeft > 0) {
+ final int numToRead;
+ if (bytesLeft < buffer.length)
+ numToRead = (int) bytesLeft;
+ else
+ numToRead = buffer.length;
+ input.readBytes(buffer, 0, numToRead, false);
+ bytesLeft -= numToRead;
+ }
+ // Don't do this in your real backups! This is just
+ // to force a backup to take a somewhat long time, to
+ // make sure we are exercising the fact that the
+ // IndexWriter should not delete this file even when I
+ // take my time reading it.
+ Thread.sleep(1);
+ } finally {
+ input.close();
+ }
+ }
+
+
+ @Test
+ public void testBasicSnapshots() throws Exception {
+ int numSnapshots = 3;
+ SnapshotDeletionPolicy sdp = getDeletionPolicy();
+
+ // Create 3 snapshots: snapshot0, snapshot1, snapshot2
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
+ prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
+ writer.close();
+
+ assertSnapshotExists(dir, sdp, numSnapshots);
+
+ // open a reader on a snapshot - should succeed.
+ IndexReader.open(sdp.getSnapshot("snapshot0"), true).close();
+
+ // open a new IndexWriter w/ no snapshots to keep and assert that all snapshots are gone.
+ sdp = getDeletionPolicy();
+ writer = new IndexWriter(dir, getConfig(random, sdp));
+ writer.deleteUnusedFiles();
+ writer.close();
+ assertEquals("no snapshots should exist", 1, IndexReader.listCommits(dir).size());
+
+ for (int i = 0; i < numSnapshots; i++) {
+ try {
+ sdp.getSnapshot("snapshot" + i);
+ fail("snapshot shouldn't have existed, but did: snapshot" + i);
+ } catch (IllegalStateException e) {
+ // expected - snapshot should not exist
+ }
+ }
+ dir.close();
+ }
+
+ @Test
+ public void testMultiThreadedSnapshotting() throws Exception {
+ Directory dir = newDirectory();
+ final SnapshotDeletionPolicy sdp = getDeletionPolicy();
+ final IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
+
+ Thread[] threads = new Thread[10];
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new Thread() {
+ @Override
+ public void run() {
+ try {
+ writer.addDocument(new Document());
+ writer.commit();
+ sdp.snapshot(getName());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+ threads[i].setName("t" + i);
+ }
+
+ for (Thread t : threads) {
+ t.start();
+ }
+
+ for (Thread t : threads) {
+ t.join();
+ }
+
+ // Do one last commit, so that after we release all snapshots, we stay w/ one commit
+ writer.addDocument(new Document());
+ writer.commit();
+
+ for (Thread t : threads) {
+ sdp.release(t.getName());
+ writer.deleteUnusedFiles();
+ }
+ assertEquals(1, IndexReader.listCommits(dir).size());
+ writer.close();
+ dir.close();
+ }
+
+ @Test
+ public void testRollbackToOldSnapshot() throws Exception {
+ int numSnapshots = 2;
+ Directory dir = newDirectory();
+ SnapshotDeletionPolicy sdp = getDeletionPolicy();
+ IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
+ prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
+ writer.close();
+
+ // now open the writer on "snapshot0" - make sure it succeeds
+ writer = new IndexWriter(dir, getConfig(random, sdp).setIndexCommit(sdp.getSnapshot("snapshot0")));
+ // this does the actual rollback
+ writer.commit();
+ writer.deleteUnusedFiles();
+ assertSnapshotExists(dir, sdp, numSnapshots - 1);
+ writer.close();
+
+ // but 'snapshot1' files will still exist (need to release snapshot before they can be deleted).
+ String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName();
+ assertTrue("snapshot files should exist in the directory: " + segFileName, dir.fileExists(segFileName));
+ dir.close();
+ }
+
+ @Test
+ public void testReleaseSnapshot() throws Exception {
+ Directory dir = newDirectory();
+ SnapshotDeletionPolicy sdp = getDeletionPolicy();
+ IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
+ prepareIndexAndSnapshots(sdp, writer, 1, "snapshot");
+
+ // Create another commit - we must do that, because otherwise the "snapshot"
+ // files will still remain in the index, since it's the last commit.
+ writer.addDocument(new Document());
+ writer.commit();
+
+ // Release
+ String snapId = "snapshot0";
+ String segFileName = sdp.getSnapshot(snapId).getSegmentsFileName();
+ sdp.release(snapId);
+ try {
+ sdp.getSnapshot(snapId);
+ fail("should not have succeeded to get an unsnapshotted id");
+ } catch (IllegalStateException e) {
+ // expected
+ }
+ assertNull(sdp.getSnapshots().get(snapId));
+ writer.deleteUnusedFiles();
+ writer.close();
+ assertFalse("segments file should not be found in dirctory: " + segFileName, dir.fileExists(segFileName));
+ dir.close();
+ }
+
+ @Test
+ public void testExistingSnapshots() throws Exception {
+ // Tests the ability to construct a SDP from existing snapshots, and
+ // asserts that those snapshots/commit points are protected.
+ int numSnapshots = 3;
+ Directory dir = newDirectory();
+ SnapshotDeletionPolicy sdp = getDeletionPolicy();
+ IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
+ prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
+ writer.close();
+
+ // Make a new policy and initialize with snapshots.
+ sdp = getDeletionPolicy(sdp.getSnapshots());
+ writer = new IndexWriter(dir, getConfig(random, sdp));
+ // attempt to delete unused files - the snapshotted files should not be deleted
+ writer.deleteUnusedFiles();
+ writer.close();
+ assertSnapshotExists(dir, sdp, numSnapshots);
+ dir.close();
+ }
+
+ @Test
+ public void testSnapshotLastCommitTwice() throws Exception {
+ Directory dir = newDirectory();
+ SnapshotDeletionPolicy sdp = getDeletionPolicy();
+ IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
+ writer.addDocument(new Document());
+ writer.commit();
+
+ String s1 = "s1";
+ String s2 = "s2";
+ IndexCommit ic1 = sdp.snapshot(s1);
+ IndexCommit ic2 = sdp.snapshot(s2);
+ assertTrue(ic1 == ic2); // should be the same instance
+
+ // create another commit
+ writer.addDocument(new Document());
+ writer.commit();
+
+ // release "s1" should not delete "s2"
+ sdp.release(s1);
+ writer.deleteUnusedFiles();
+ checkSnapshotExists(dir, ic2);
+
+ writer.close();
+ dir.close();
+ }
+
+ @Test
+ public void testMissingCommits() throws Exception {
+ // Tests the behavior of SDP when commits that are given at ctor are missing
+ // on onInit().
+ Directory dir = newDirectory();
+ SnapshotDeletionPolicy sdp = getDeletionPolicy();
+ IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
+ writer.addDocument(new Document());
+ writer.commit();
+ IndexCommit ic = sdp.snapshot("s1");
+
+ // create another commit, not snapshotted.
+ writer.addDocument(new Document());
+ writer.close();
+
+ // open a new writer w/ KeepOnlyLastCommit policy, so it will delete "s1"
+ // commit.
+ new IndexWriter(dir, getConfig(random, null)).close();
+
+ assertFalse("snapshotted commit should not exist", dir.fileExists(ic.getSegmentsFileName()));
+
+ // Now reinit SDP from the commits in the index - the snapshot id should not
+ // exist anymore.
+ sdp = getDeletionPolicy(sdp.getSnapshots());
+ new IndexWriter(dir, getConfig(random, sdp)).close();
+
+ try {
+ sdp.getSnapshot("s1");
+ fail("snapshot s1 should not exist");
+ } catch (IllegalStateException e) {
+ // expected.
+ }
+ dir.close();
+ }
+
+}
fnp / pylucene.git / blobdiff