--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.Collection;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.ScoreDoc;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+/*
+ Verify we can read the pre-2.1 file format, do searches
+ against it, and add documents to it.
+*/
+
+public class TestDeletionPolicy extends LuceneTestCase {
+
+ private void verifyCommitOrder(List<? extends IndexCommit> commits) throws IOException {
+ final IndexCommit firstCommit = commits.get(0);
+ long last = SegmentInfos.generationFromSegmentsFileName(firstCommit.getSegmentsFileName());
+ assertEquals(last, firstCommit.getGeneration());
+ long lastVersion = firstCommit.getVersion();
+ long lastTimestamp = firstCommit.getTimestamp();
+ for(int i=1;i<commits.size();i++) {
+ final IndexCommit commit = commits.get(i);
+ long now = SegmentInfos.generationFromSegmentsFileName(commit.getSegmentsFileName());
+ long nowVersion = commit.getVersion();
+ long nowTimestamp = commit.getTimestamp();
+ assertTrue("SegmentInfos commits are out-of-order", now > last);
+ assertTrue("SegmentInfos versions are out-of-order", nowVersion > lastVersion);
+ assertTrue("SegmentInfos timestamps are out-of-order: now=" + nowTimestamp + " vs last=" + lastTimestamp, nowTimestamp >= lastTimestamp);
+ assertEquals(now, commit.getGeneration());
+ last = now;
+ lastVersion = nowVersion;
+ lastTimestamp = nowTimestamp;
+ }
+ }
+
+ class KeepAllDeletionPolicy implements IndexDeletionPolicy {
+ int numOnInit;
+ int numOnCommit;
+ Directory dir;
+ public void onInit(List<? extends IndexCommit> commits) throws IOException {
+ verifyCommitOrder(commits);
+ numOnInit++;
+ }
+ public void onCommit(List<? extends IndexCommit> commits) throws IOException {
+ IndexCommit lastCommit = commits.get(commits.size()-1);
+ IndexReader r = IndexReader.open(dir, true);
+ assertEquals("lastCommit.segmentCount()=" + lastCommit.getSegmentCount() + " vs IndexReader.segmentCount=" + r.getSequentialSubReaders().length, r.getSequentialSubReaders().length, lastCommit.getSegmentCount());
+ r.close();
+ verifyCommitOrder(commits);
+ numOnCommit++;
+ }
+ }
+
+ /**
+ * This is useful for adding to a big index when you know
+ * readers are not using it.
+ */
+ class KeepNoneOnInitDeletionPolicy implements IndexDeletionPolicy {
+ int numOnInit;
+ int numOnCommit;
+ public void onInit(List<? extends IndexCommit> commits) throws IOException {
+ verifyCommitOrder(commits);
+ numOnInit++;
+ // On init, delete all commit points:
+ for (final IndexCommit commit : commits) {
+ commit.delete();
+ assertTrue(commit.isDeleted());
+ }
+ }
+ public void onCommit(List<? extends IndexCommit> commits) throws IOException {
+ verifyCommitOrder(commits);
+ int size = commits.size();
+ // Delete all but last one:
+ for(int i=0;i<size-1;i++) {
+ ((IndexCommit) commits.get(i)).delete();
+ }
+ numOnCommit++;
+ }
+ }
+
+ class KeepLastNDeletionPolicy implements IndexDeletionPolicy {
+ int numOnInit;
+ int numOnCommit;
+ int numToKeep;
+ int numDelete;
+ Set<String> seen = new HashSet<String>();
+
+ public KeepLastNDeletionPolicy(int numToKeep) {
+ this.numToKeep = numToKeep;
+ }
+
+ public void onInit(List<? extends IndexCommit> commits) throws IOException {
+ if (VERBOSE) {
+ System.out.println("TEST: onInit");
+ }
+ verifyCommitOrder(commits);
+ numOnInit++;
+ // do no deletions on init
+ doDeletes(commits, false);
+ }
+
+ public void onCommit(List<? extends IndexCommit> commits) throws IOException {
+ if (VERBOSE) {
+ System.out.println("TEST: onCommit");
+ }
+ verifyCommitOrder(commits);
+ doDeletes(commits, true);
+ }
+
+ private void doDeletes(List<? extends IndexCommit> commits, boolean isCommit) {
+
+ // Assert that we really are only called for each new
+ // commit:
+ if (isCommit) {
+ String fileName = ((IndexCommit) commits.get(commits.size()-1)).getSegmentsFileName();
+ if (seen.contains(fileName)) {
+ throw new RuntimeException("onCommit was called twice on the same commit point: " + fileName);
+ }
+ seen.add(fileName);
+ numOnCommit++;
+ }
+ int size = commits.size();
+ for(int i=0;i<size-numToKeep;i++) {
+ ((IndexCommit) commits.get(i)).delete();
+ numDelete++;
+ }
+ }
+ }
+
+ /*
+ * Delete a commit only when it has been obsoleted by N
+ * seconds.
+ */
+ class ExpirationTimeDeletionPolicy implements IndexDeletionPolicy {
+
+ Directory dir;
+ double expirationTimeSeconds;
+ int numDelete;
+
+ public ExpirationTimeDeletionPolicy(Directory dir, double seconds) {
+ this.dir = dir;
+ this.expirationTimeSeconds = seconds;
+ }
+
+ public void onInit(List<? extends IndexCommit> commits) throws IOException {
+ verifyCommitOrder(commits);
+ onCommit(commits);
+ }
+
+ public void onCommit(List<? extends IndexCommit> commits) throws IOException {
+ verifyCommitOrder(commits);
+
+ IndexCommit lastCommit = commits.get(commits.size()-1);
+
+ // Any commit older than expireTime should be deleted:
+ double expireTime = dir.fileModified(lastCommit.getSegmentsFileName())/1000.0 - expirationTimeSeconds;
+
+ for (final IndexCommit commit : commits) {
+ double modTime = dir.fileModified(commit.getSegmentsFileName())/1000.0;
+ if (commit != lastCommit && modTime < expireTime) {
+ commit.delete();
+ numDelete += 1;
+ }
+ }
+ }
+ }
+
+ /*
+ * Test "by time expiration" deletion policy:
+ */
+ public void testExpirationTimeDeletionPolicy() throws IOException, InterruptedException {
+
+ final double SECONDS = 2.0;
+
+ Directory dir = newDirectory();
+ ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS);
+ IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random))
+ .setIndexDeletionPolicy(policy);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, true);
+ }
+ IndexWriter writer = new IndexWriter(dir, conf);
+ writer.close();
+
+ final int ITER = 9;
+
+ long lastDeleteTime = 0;
+ for(int i=0;i<ITER;i++) {
+ // Record last time when writer performed deletes of
+ // past commits
+ lastDeleteTime = System.currentTimeMillis();
+ conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random)).setOpenMode(
+ OpenMode.APPEND).setIndexDeletionPolicy(policy);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, true);
+ }
+ writer = new IndexWriter(dir, conf);
+ for(int j=0;j<17;j++) {
+ addDoc(writer);
+ }
+ writer.close();
+
+ if (i < ITER-1) {
+ // Make sure to sleep long enough so that some commit
+ // points will be deleted:
+ Thread.sleep((int) (1000.0*(SECONDS/5.0)));
+ }
+ }
+
+ // First, make sure the policy in fact deleted something:
+ assertTrue("no commits were deleted", policy.numDelete > 0);
+
+ // Then simplistic check: just verify that the
+ // segments_N's that still exist are in fact within SECONDS
+ // seconds of the last one's mod time, and, that I can
+ // open a reader on each:
+ long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+
+ String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ gen);
+ dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
+
+ boolean oneSecondResolution = true;
+
+ while(gen > 0) {
+ try {
+ IndexReader reader = IndexReader.open(dir, true);
+ reader.close();
+ fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+ "",
+ gen);
+
+ // if we are on a filesystem that seems to have only
+ // 1 second resolution, allow +1 second in commit
+ // age tolerance:
+ long modTime = dir.fileModified(fileName);
+ oneSecondResolution &= (modTime % 1000) == 0;
+ final long leeway = (long) ((SECONDS + (oneSecondResolution ? 1.0:0.0))*1000);
+
+ assertTrue("commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted ", lastDeleteTime - modTime <= leeway);
+ } catch (IOException e) {
+ // OK
+ break;
+ }
+
+ dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ gen--;
+ }
+
+ dir.close();
+ }
+
+ /*
+ * Test a silly deletion policy that keeps all commits around.
+ */
+ public void testKeepAllDeletionPolicy() throws IOException {
+ for(int pass=0;pass<2;pass++) {
+
+ if (VERBOSE) {
+ System.out.println("TEST: cycle pass=" + pass);
+ }
+
+ boolean useCompoundFile = (pass % 2) != 0;
+
+ // Never deletes a commit
+ KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();
+
+ Directory dir = newDirectory();
+ policy.dir = dir;
+
+ IndexWriterConfig conf = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setIndexDeletionPolicy(policy).setMaxBufferedDocs(10)
+ .setMergeScheduler(new SerialMergeScheduler());
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, useCompoundFile);
+ }
+ IndexWriter writer = new IndexWriter(dir, conf);
+ for(int i=0;i<107;i++) {
+ addDoc(writer);
+ }
+ writer.close();
+
+ final boolean needsMerging;
+ {
+ IndexReader r = IndexReader.open(dir);
+ needsMerging = r.getSequentialSubReaders().length != 1;
+ r.close();
+ }
+ if (needsMerging) {
+ conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(
+ OpenMode.APPEND).setIndexDeletionPolicy(policy);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, true);
+ }
+ if (VERBOSE) {
+ System.out.println("TEST: open writer for forceMerge");
+ }
+ writer = new IndexWriter(dir, conf);
+ writer.setInfoStream(VERBOSE ? System.out : null);
+ writer.forceMerge(1);
+ writer.close();
+ }
+ assertEquals(needsMerging ? 1:0, policy.numOnInit);
+
+ // If we are not auto committing then there should
+ // be exactly 2 commits (one per close above):
+ assertEquals(1 + (needsMerging ? 1:0), policy.numOnCommit);
+
+ // Test listCommits
+ Collection<IndexCommit> commits = IndexReader.listCommits(dir);
+ // 2 from closing writer
+ assertEquals(1 + (needsMerging ? 1:0), commits.size());
+
+ // Make sure we can open a reader on each commit:
+ for (final IndexCommit commit : commits) {
+ IndexReader r = IndexReader.open(commit, null, false);
+ r.close();
+ }
+
+ // Simplistic check: just verify all segments_N's still
+ // exist, and, I can open a reader on each:
+ dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
+ long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+ while(gen > 0) {
+ IndexReader reader = IndexReader.open(dir, true);
+ reader.close();
+ dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ gen--;
+
+ if (gen > 0) {
+ // Now that we've removed a commit point, which
+ // should have orphan'd at least one index file.
+ // Open & close a writer and assert that it
+ // actually removed something:
+ int preCount = dir.listAll().length;
+ writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT,
+ new MockAnalyzer(random)).setOpenMode(
+ OpenMode.APPEND).setIndexDeletionPolicy(policy));
+ writer.close();
+ int postCount = dir.listAll().length;
+ assertTrue(postCount < preCount);
+ }
+ }
+
+ dir.close();
+ }
+ }
+
+ /* Uses KeepAllDeletionPolicy to keep all commits around,
+ * then, opens a new IndexWriter on a previous commit
+ * point. */
+ public void testOpenPriorSnapshot() throws IOException {
+ // Never deletes a commit
+ KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();
+
+ Directory dir = newDirectory();
+ policy.dir = dir;
+
+ IndexWriter writer = new IndexWriter(
+ dir,
+ newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
+ setIndexDeletionPolicy(policy).
+ setMaxBufferedDocs(2).
+ setMergePolicy(newLogMergePolicy(10))
+ );
+ for(int i=0;i<10;i++) {
+ addDoc(writer);
+ if ((1+i)%2 == 0)
+ writer.commit();
+ }
+ writer.close();
+
+ Collection<IndexCommit> commits = IndexReader.listCommits(dir);
+ assertEquals(5, commits.size());
+ IndexCommit lastCommit = null;
+ for (final IndexCommit commit : commits) {
+ if (lastCommit == null || commit.getGeneration() > lastCommit.getGeneration())
+ lastCommit = commit;
+ }
+ assertTrue(lastCommit != null);
+
+ // Now add 1 doc and merge
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
+ addDoc(writer);
+ assertEquals(11, writer.numDocs());
+ writer.forceMerge(1);
+ writer.close();
+
+ assertEquals(6, IndexReader.listCommits(dir).size());
+
+ // Now open writer on the commit just before merge:
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
+ assertEquals(10, writer.numDocs());
+
+ // Should undo our rollback:
+ writer.rollback();
+
+ IndexReader r = IndexReader.open(dir, true);
+ // Still merged, still 11 docs
+ assertEquals(1, r.getSequentialSubReaders().length);
+ assertEquals(11, r.numDocs());
+ r.close();
+
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
+ assertEquals(10, writer.numDocs());
+ // Commits the rollback:
+ writer.close();
+
+ // Now 8 because we made another commit
+ assertEquals(7, IndexReader.listCommits(dir).size());
+
+ r = IndexReader.open(dir, true);
+ // Not fully merged because we rolled it back, and now only
+ // 10 docs
+ assertTrue(r.getSequentialSubReaders().length > 1);
+ assertEquals(10, r.numDocs());
+ r.close();
+
+ // Re-merge
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
+ writer.forceMerge(1);
+ writer.close();
+
+ r = IndexReader.open(dir, true);
+ assertEquals(1, r.getSequentialSubReaders().length);
+ assertEquals(10, r.numDocs());
+ r.close();
+
+ // Now open writer on the commit just before merging,
+ // but this time keeping only the last commit:
+ writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexCommit(lastCommit));
+ assertEquals(10, writer.numDocs());
+
+ // Reader still sees fully merged index, because writer
+ // opened on the prior commit has not yet committed:
+ r = IndexReader.open(dir, true);
+ assertEquals(1, r.getSequentialSubReaders().length);
+ assertEquals(10, r.numDocs());
+ r.close();
+
+ writer.close();
+
+ // Now reader sees not-fully-merged index:
+ r = IndexReader.open(dir, true);
+ assertTrue(r.getSequentialSubReaders().length > 1);
+ assertEquals(10, r.numDocs());
+ r.close();
+
+ dir.close();
+ }
+
+
+ /* Test keeping NO commit points. This is a viable and
+ * useful case eg where you want to build a big index and
+ * you know there are no readers.
+ */
+ public void testKeepNoneOnInitDeletionPolicy() throws IOException {
+ for(int pass=0;pass<2;pass++) {
+
+ boolean useCompoundFile = (pass % 2) != 0;
+
+ KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy();
+
+ Directory dir = newDirectory();
+
+ IndexWriterConfig conf = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
+ .setMaxBufferedDocs(10);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, useCompoundFile);
+ }
+ IndexWriter writer = new IndexWriter(dir, conf);
+ for(int i=0;i<107;i++) {
+ addDoc(writer);
+ }
+ writer.close();
+
+ conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, true);
+ }
+ writer = new IndexWriter(dir, conf);
+ writer.forceMerge(1);
+ writer.close();
+
+ assertEquals(1, policy.numOnInit);
+ // If we are not auto committing then there should
+ // be exactly 2 commits (one per close above):
+ assertEquals(2, policy.numOnCommit);
+
+ // Simplistic check: just verify the index is in fact
+ // readable:
+ IndexReader reader = IndexReader.open(dir, true);
+ reader.close();
+
+ dir.close();
+ }
+ }
+
+ /*
+ * Test a deletion policy that keeps last N commits.
+ */
+ public void testKeepLastNDeletionPolicy() throws IOException {
+ final int N = 5;
+
+ for(int pass=0;pass<2;pass++) {
+
+ boolean useCompoundFile = (pass % 2) != 0;
+
+ Directory dir = newDirectory();
+
+ KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
+
+ for(int j=0;j<N+1;j++) {
+ IndexWriterConfig conf = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
+ .setMaxBufferedDocs(10);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, useCompoundFile);
+ }
+ IndexWriter writer = new IndexWriter(dir, conf);
+ for(int i=0;i<17;i++) {
+ addDoc(writer);
+ }
+ writer.forceMerge(1);
+ writer.close();
+ }
+
+ assertTrue(policy.numDelete > 0);
+ assertEquals(N, policy.numOnInit);
+ assertEquals(N+1, policy.numOnCommit);
+
+ // Simplistic check: just verify only the past N segments_N's still
+ // exist, and, I can open a reader on each:
+ dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
+ long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+ for(int i=0;i<N+1;i++) {
+ try {
+ IndexReader reader = IndexReader.open(dir, true);
+ reader.close();
+ if (i == N) {
+ fail("should have failed on commits prior to last " + N);
+ }
+ } catch (IOException e) {
+ if (i != N) {
+ throw e;
+ }
+ }
+ if (i < N) {
+ dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ }
+ gen--;
+ }
+
+ dir.close();
+ }
+ }
+
+ /*
+ * Test a deletion policy that keeps last N commits
+ * around, with reader doing deletes.
+ */
+ public void testKeepLastNDeletionPolicyWithReader() throws IOException {
+ final int N = 10;
+
+ for(int pass=0;pass<2;pass++) {
+
+ boolean useCompoundFile = (pass % 2) != 0;
+
+ KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
+
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, useCompoundFile);
+ }
+ IndexWriter writer = new IndexWriter(dir, conf);
+ writer.close();
+ Term searchTerm = new Term("content", "aaa");
+ Query query = new TermQuery(searchTerm);
+
+ for(int i=0;i<N+1;i++) {
+ if (VERBOSE) {
+ System.out.println("\nTEST: cycle i=" + i);
+ }
+ conf = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, useCompoundFile);
+ }
+ writer = new IndexWriter(dir, conf);
+ writer.setInfoStream(VERBOSE ? System.out : null);
+ for(int j=0;j<17;j++) {
+ addDoc(writer);
+ }
+ // this is a commit
+ if (VERBOSE) {
+ System.out.println("TEST: close writer");
+ }
+ writer.close();
+ IndexReader reader = IndexReader.open(dir, policy, false);
+ reader.deleteDocument(3*i+1);
+ reader.setNorm(4*i+1, "content", 2.0F);
+ IndexSearcher searcher = newSearcher(reader);
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(16*(1+i), hits.length);
+ // this is a commit
+ if (VERBOSE) {
+ System.out.println("TEST: close reader numOnCommit=" + policy.numOnCommit);
+ }
+ reader.close();
+ searcher.close();
+ }
+ conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, useCompoundFile);
+ }
+ IndexReader r = IndexReader.open(dir);
+ final boolean wasFullyMerged = r.getSequentialSubReaders().length == 1 && !r.hasDeletions();
+ r.close();
+ writer = new IndexWriter(dir, conf);
+ writer.forceMerge(1);
+ // this is a commit
+ writer.close();
+
+ assertEquals(2*(N+1)+1, policy.numOnInit);
+ assertEquals(2*(N+2) - (wasFullyMerged ? 1:0), policy.numOnCommit);
+
+ IndexReader rwReader = IndexReader.open(dir, false);
+ IndexSearcher searcher = new IndexSearcher(rwReader);
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(176, hits.length);
+
+ // Simplistic check: just verify only the past N segments_N's still
+ // exist, and, I can open a reader on each:
+ long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+
+ dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
+ int expectedCount = 176;
+ searcher.close();
+ rwReader.close();
+ for(int i=0;i<N+1;i++) {
+ try {
+ IndexReader reader = IndexReader.open(dir, true);
+
+ // Work backwards in commits on what the expected
+ // count should be.
+ searcher = newSearcher(reader);
+ hits = searcher.search(query, null, 1000).scoreDocs;
+ if (i > 1) {
+ if (i % 2 == 0) {
+ expectedCount += 1;
+ } else {
+ expectedCount -= 17;
+ }
+ }
+ assertEquals(expectedCount, hits.length);
+ searcher.close();
+ reader.close();
+ if (i == N) {
+ fail("should have failed on commits before last 5");
+ }
+ } catch (IOException e) {
+ if (i != N) {
+ throw e;
+ }
+ }
+ if (i < N) {
+ dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ }
+ gen--;
+ }
+ dir.close();
+ }
+ }
+
+ /*
+ * Test a deletion policy that keeps last N commits
+ * around, through creates.
+ */
+ public void testKeepLastNDeletionPolicyWithCreates() throws IOException {
+
+ final int N = 10;
+
+ for(int pass=0;pass<2;pass++) {
+
+ boolean useCompoundFile = (pass % 2) != 0;
+
+ KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
+
+ Directory dir = newDirectory();
+ IndexWriterConfig conf = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
+ .setMaxBufferedDocs(10);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, useCompoundFile);
+ }
+ IndexWriter writer = new IndexWriter(dir, conf);
+ writer.close();
+ Term searchTerm = new Term("content", "aaa");
+ Query query = new TermQuery(searchTerm);
+
+ for(int i=0;i<N+1;i++) {
+
+ conf = newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy)
+ .setMaxBufferedDocs(10);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ setUseCompoundFile(mp, useCompoundFile);
+ }
+ writer = new IndexWriter(dir, conf);
+ for(int j=0;j<17;j++) {
+ addDoc(writer);
+ }
+ // this is a commit
+ writer.close();
+ IndexReader reader = IndexReader.open(dir, policy, false);
+ reader.deleteDocument(3);
+ reader.setNorm(5, "content", 2.0F);
+ IndexSearcher searcher = newSearcher(reader);
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(16, hits.length);
+ // this is a commit
+ reader.close();
+ searcher.close();
+
+ writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random))
+ .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy));
+ // This will not commit: there are no changes
+ // pending because we opened for "create":
+ writer.close();
+ }
+
+ assertEquals(3*(N+1), policy.numOnInit);
+ assertEquals(3*(N+1)+1, policy.numOnCommit);
+
+ IndexReader rwReader = IndexReader.open(dir, false);
+ IndexSearcher searcher = new IndexSearcher(rwReader);
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(0, hits.length);
+
+ // Simplistic check: just verify only the past N segments_N's still
+ // exist, and, I can open a reader on each:
+ long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
+
+ dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
+ int expectedCount = 0;
+
+ searcher.close();
+ rwReader.close();
+
+ for(int i=0;i<N+1;i++) {
+ try {
+ IndexReader reader = IndexReader.open(dir, true);
+
+ // Work backwards in commits on what the expected
+ // count should be.
+ searcher = newSearcher(reader);
+ hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(expectedCount, hits.length);
+ searcher.close();
+ if (expectedCount == 0) {
+ expectedCount = 16;
+ } else if (expectedCount == 16) {
+ expectedCount = 17;
+ } else if (expectedCount == 17) {
+ expectedCount = 0;
+ }
+ reader.close();
+ if (i == N) {
+ fail("should have failed on commits before last " + N);
+ }
+ } catch (IOException e) {
+ if (i != N) {
+ throw e;
+ }
+ }
+ if (i < N) {
+ dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
+ }
+ gen--;
+ }
+
+ dir.close();
+ }
+ }
+
+ private void addDoc(IndexWriter writer) throws IOException
+ {
+ Document doc = new Document();
+ doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ }
+}
fnp / pylucene.git / blobdiff