1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.util.Collection;
22 import java.util.Random;
23 import java.io.IOException;
25 import org.apache.lucene.document.Document;
26 import org.apache.lucene.document.Field;
27 import org.apache.lucene.store.Directory;
28 import org.apache.lucene.store.IndexInput;
29 import org.apache.lucene.analysis.KeywordAnalyzer;
30 import org.apache.lucene.analysis.MockAnalyzer;
31 import org.apache.lucene.analysis.standard.StandardAnalyzer;
32 import org.apache.lucene.index.IndexCommit;
33 import org.apache.lucene.index.IndexWriterConfig;
34 import org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy;
35 import org.apache.lucene.index.IndexWriter;
36 import org.apache.lucene.index.SnapshotDeletionPolicy;
37 import org.apache.lucene.util.LuceneTestCase;
38 import org.apache.lucene.util.ThreadInterruptedException;
39 import org.junit.Test;
42 // This was developed for Lucene In Action,
43 // http://lucenebook.com
46 public class TestSnapshotDeletionPolicy extends LuceneTestCase {
47 public static final String INDEX_PATH = "test.snapshots";
49 protected IndexWriterConfig getConfig(Random random, IndexDeletionPolicy dp) {
50 IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random));
52 conf.setIndexDeletionPolicy(dp);
57 protected void checkSnapshotExists(Directory dir, IndexCommit c) throws Exception {
58 String segFileName = c.getSegmentsFileName();
59 assertTrue("segments file not found in directory: " + segFileName, dir.fileExists(segFileName));
62 protected void checkMaxDoc(IndexCommit commit, int expectedMaxDoc) throws Exception {
63 IndexReader reader = IndexReader.open(commit, true);
65 assertEquals(expectedMaxDoc, reader.maxDoc());
71 protected void prepareIndexAndSnapshots(SnapshotDeletionPolicy sdp,
72 IndexWriter writer, int numSnapshots, String snapshotPrefix)
73 throws RuntimeException, IOException {
74 for (int i = 0; i < numSnapshots; i++) {
75 // create dummy document to trigger commit.
76 writer.addDocument(new Document());
78 sdp.snapshot(snapshotPrefix + i);
82 protected SnapshotDeletionPolicy getDeletionPolicy() throws IOException {
83 return getDeletionPolicy(null);
86 protected SnapshotDeletionPolicy getDeletionPolicy(Map<String, String> snapshots) throws IOException {
87 return new SnapshotDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy(), snapshots);
90 protected void assertSnapshotExists(Directory dir, SnapshotDeletionPolicy sdp, int numSnapshots) throws Exception {
91 for (int i = 0; i < numSnapshots; i++) {
92 IndexCommit snapshot = sdp.getSnapshot("snapshot" + i);
93 checkMaxDoc(snapshot, i + 1);
94 checkSnapshotExists(dir, snapshot);
99 public void testSnapshotDeletionPolicy() throws Exception {
100 Directory fsDir = newDirectory();
101 runTest(random, fsDir);
105 private void runTest(Random random, Directory dir) throws Exception {
106 // Run for ~1 seconds
107 final long stopTime = System.currentTimeMillis() + 1000;
109 SnapshotDeletionPolicy dp = getDeletionPolicy();
110 final IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
111 TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(dp)
112 .setMaxBufferedDocs(2));
115 final Thread t = new Thread() {
118 Document doc = new Document();
119 doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
121 for(int i=0;i<27;i++) {
123 writer.addDocument(doc);
124 } catch (Throwable t) {
125 t.printStackTrace(System.out);
126 fail("addDocument failed");
131 } catch (Exception e) {
132 throw new RuntimeException(e);
138 } catch (InterruptedException ie) {
139 throw new ThreadInterruptedException(ie);
141 } while(System.currentTimeMillis() < stopTime);
147 // While the above indexing thread is running, take many
150 backupIndex(dir, dp);
152 } while(t.isAlive());
156 // Add one more document to force writer to commit a
157 // final segment, so deletion policy has a chance to
159 Document doc = new Document();
160 doc.add(newField("content", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
161 writer.addDocument(doc);
163 // Make sure we don't have any leftover files in the
166 TestIndexWriter.assertNoUnreferencedFiles(dir, "some files were not deleted but should have been");
170 * Example showing how to use the SnapshotDeletionPolicy to take a backup.
171 * This method does not really do a backup; instead, it reads every byte of
172 * every file just to test that the files indeed exist and are readable even
173 * while the index is changing.
175 public void backupIndex(Directory dir, SnapshotDeletionPolicy dp) throws Exception {
176 // To backup an index we first take a snapshot:
178 copyFiles(dir, dp.snapshot("id"));
180 // Make sure to release the snapshot, otherwise these
181 // files will never be deleted during this IndexWriter
187 private void copyFiles(Directory dir, IndexCommit cp) throws Exception {
189 // While we hold the snapshot, and nomatter how long
190 // we take to do the backup, the IndexWriter will
191 // never delete the files in the snapshot:
192 Collection<String> files = cp.getFileNames();
193 for (final String fileName : files) {
194 // NOTE: in a real backup you would not use
195 // readFile; you would need to use something else
196 // that copies the file to a backup location. This
197 // could even be a spawned shell process (eg "tar",
198 // "zip") that takes the list of files and builds a
200 readFile(dir, fileName);
204 byte[] buffer = new byte[4096];
206 private void readFile(Directory dir, String name) throws Exception {
207 IndexInput input = dir.openInput(name);
209 long size = dir.fileLength(name);
210 long bytesLeft = size;
211 while (bytesLeft > 0) {
213 if (bytesLeft < buffer.length)
214 numToRead = (int) bytesLeft;
216 numToRead = buffer.length;
217 input.readBytes(buffer, 0, numToRead, false);
218 bytesLeft -= numToRead;
220 // Don't do this in your real backups! This is just
221 // to force a backup to take a somewhat long time, to
222 // make sure we are exercising the fact that the
223 // IndexWriter should not delete this file even when I
224 // take my time reading it.
233 public void testBasicSnapshots() throws Exception {
234 int numSnapshots = 3;
235 SnapshotDeletionPolicy sdp = getDeletionPolicy();
237 // Create 3 snapshots: snapshot0, snapshot1, snapshot2
238 Directory dir = newDirectory();
239 IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
240 prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
243 assertSnapshotExists(dir, sdp, numSnapshots);
245 // open a reader on a snapshot - should succeed.
246 IndexReader.open(sdp.getSnapshot("snapshot0"), true).close();
248 // open a new IndexWriter w/ no snapshots to keep and assert that all snapshots are gone.
249 sdp = getDeletionPolicy();
250 writer = new IndexWriter(dir, getConfig(random, sdp));
251 writer.deleteUnusedFiles();
253 assertEquals("no snapshots should exist", 1, IndexReader.listCommits(dir).size());
255 for (int i = 0; i < numSnapshots; i++) {
257 sdp.getSnapshot("snapshot" + i);
258 fail("snapshot shouldn't have existed, but did: snapshot" + i);
259 } catch (IllegalStateException e) {
260 // expected - snapshot should not exist
267 public void testMultiThreadedSnapshotting() throws Exception {
268 Directory dir = newDirectory();
269 final SnapshotDeletionPolicy sdp = getDeletionPolicy();
270 final IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
272 Thread[] threads = new Thread[10];
273 for (int i = 0; i < threads.length; i++) {
274 threads[i] = new Thread() {
278 writer.addDocument(new Document());
280 sdp.snapshot(getName());
281 } catch (Exception e) {
282 throw new RuntimeException(e);
286 threads[i].setName("t" + i);
289 for (Thread t : threads) {
293 for (Thread t : threads) {
297 // Do one last commit, so that after we release all snapshots, we stay w/ one commit
298 writer.addDocument(new Document());
301 for (Thread t : threads) {
302 sdp.release(t.getName());
303 writer.deleteUnusedFiles();
305 assertEquals(1, IndexReader.listCommits(dir).size());
311 public void testRollbackToOldSnapshot() throws Exception {
312 int numSnapshots = 2;
313 Directory dir = newDirectory();
314 SnapshotDeletionPolicy sdp = getDeletionPolicy();
315 IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
316 prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
319 // now open the writer on "snapshot0" - make sure it succeeds
320 writer = new IndexWriter(dir, getConfig(random, sdp).setIndexCommit(sdp.getSnapshot("snapshot0")));
321 // this does the actual rollback
323 writer.deleteUnusedFiles();
324 assertSnapshotExists(dir, sdp, numSnapshots - 1);
327 // but 'snapshot1' files will still exist (need to release snapshot before they can be deleted).
328 String segFileName = sdp.getSnapshot("snapshot1").getSegmentsFileName();
329 assertTrue("snapshot files should exist in the directory: " + segFileName, dir.fileExists(segFileName));
334 public void testReleaseSnapshot() throws Exception {
335 Directory dir = newDirectory();
336 SnapshotDeletionPolicy sdp = getDeletionPolicy();
337 IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
338 prepareIndexAndSnapshots(sdp, writer, 1, "snapshot");
340 // Create another commit - we must do that, because otherwise the "snapshot"
341 // files will still remain in the index, since it's the last commit.
342 writer.addDocument(new Document());
346 String snapId = "snapshot0";
347 String segFileName = sdp.getSnapshot(snapId).getSegmentsFileName();
350 sdp.getSnapshot(snapId);
351 fail("should not have succeeded to get an unsnapshotted id");
352 } catch (IllegalStateException e) {
355 assertNull(sdp.getSnapshots().get(snapId));
356 writer.deleteUnusedFiles();
358 assertFalse("segments file should not be found in dirctory: " + segFileName, dir.fileExists(segFileName));
363 public void testExistingSnapshots() throws Exception {
364 // Tests the ability to construct a SDP from existing snapshots, and
365 // asserts that those snapshots/commit points are protected.
366 int numSnapshots = 3;
367 Directory dir = newDirectory();
368 SnapshotDeletionPolicy sdp = getDeletionPolicy();
369 IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
370 prepareIndexAndSnapshots(sdp, writer, numSnapshots, "snapshot");
373 // Make a new policy and initialize with snapshots.
374 sdp = getDeletionPolicy(sdp.getSnapshots());
375 writer = new IndexWriter(dir, getConfig(random, sdp));
376 // attempt to delete unused files - the snapshotted files should not be deleted
377 writer.deleteUnusedFiles();
379 assertSnapshotExists(dir, sdp, numSnapshots);
384 public void testSnapshotLastCommitTwice() throws Exception {
385 Directory dir = newDirectory();
386 SnapshotDeletionPolicy sdp = getDeletionPolicy();
387 IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
388 writer.addDocument(new Document());
393 IndexCommit ic1 = sdp.snapshot(s1);
394 IndexCommit ic2 = sdp.snapshot(s2);
395 assertTrue(ic1 == ic2); // should be the same instance
397 // create another commit
398 writer.addDocument(new Document());
401 // release "s1" should not delete "s2"
403 writer.deleteUnusedFiles();
404 checkSnapshotExists(dir, ic2);
411 public void testMissingCommits() throws Exception {
412 // Tests the behavior of SDP when commits that are given at ctor are missing
414 Directory dir = newDirectory();
415 SnapshotDeletionPolicy sdp = getDeletionPolicy();
416 IndexWriter writer = new IndexWriter(dir, getConfig(random, sdp));
417 writer.addDocument(new Document());
419 IndexCommit ic = sdp.snapshot("s1");
421 // create another commit, not snapshotted.
422 writer.addDocument(new Document());
425 // open a new writer w/ KeepOnlyLastCommit policy, so it will delete "s1"
427 new IndexWriter(dir, getConfig(random, null)).close();
429 assertFalse("snapshotted commit should not exist", dir.fileExists(ic.getSegmentsFileName()));
431 // Now reinit SDP from the commits in the index - the snapshot id should not
433 sdp = getDeletionPolicy(sdp.getSnapshots());
434 new IndexWriter(dir, getConfig(random, sdp)).close();
437 sdp.getSnapshot("s1");
438 fail("snapshot s1 should not exist");
439 } catch (IllegalStateException e) {