lucene-java-3.5.0/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java

   1 package org.apache.lucene.index;
   2
   3 /**
   4  * Licensed to the Apache Software Foundation (ASF) under one or more
   5  * contributor license agreements.  See the NOTICE file distributed with
   6  * this work for additional information regarding copyright ownership.
   7  * The ASF licenses this file to You under the Apache License, Version 2.0
   8  * (the "License"); you may not use this file except in compliance with
   9  * the License.  You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19
  20 import java.io.IOException;
  21 import java.util.HashSet;
  22 import java.util.List;
  23 import java.util.Set;
  24 import java.util.Collection;
  25
  26 import org.apache.lucene.analysis.MockAnalyzer;
  27 import org.apache.lucene.analysis.WhitespaceAnalyzer;
  28 import org.apache.lucene.document.Document;
  29 import org.apache.lucene.document.Field;
  30 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
  31 import org.apache.lucene.search.IndexSearcher;
  32 import org.apache.lucene.search.Query;
  33 import org.apache.lucene.search.ScoreDoc;
  34 import org.apache.lucene.search.TermQuery;
  35 import org.apache.lucene.store.Directory;
  36 import org.apache.lucene.util.LuceneTestCase;
  37
  38 /*
  39   Verify we can read the pre-2.1 file format, do searches
  40   against it, and add documents to it.
  41 */
  42
  43 public class TestDeletionPolicy extends LuceneTestCase {
  44
  45   private void verifyCommitOrder(List<? extends IndexCommit> commits) throws IOException {
  46     final IndexCommit firstCommit =  commits.get(0);
  47     long last = SegmentInfos.generationFromSegmentsFileName(firstCommit.getSegmentsFileName());
  48     assertEquals(last, firstCommit.getGeneration());
  49     long lastVersion = firstCommit.getVersion();
  50     long lastTimestamp = firstCommit.getTimestamp();
  51     for(int i=1;i<commits.size();i++) {
  52       final IndexCommit commit =  commits.get(i);
  53       long now = SegmentInfos.generationFromSegmentsFileName(commit.getSegmentsFileName());
  54       long nowVersion = commit.getVersion();
  55       long nowTimestamp = commit.getTimestamp();
  56       assertTrue("SegmentInfos commits are out-of-order", now > last);
  57       assertTrue("SegmentInfos versions are out-of-order", nowVersion > lastVersion);
  58       assertTrue("SegmentInfos timestamps are out-of-order: now=" + nowTimestamp + " vs last=" + lastTimestamp, nowTimestamp >= lastTimestamp);
  59       assertEquals(now, commit.getGeneration());
  60       last = now;
  61       lastVersion = nowVersion;
  62       lastTimestamp = nowTimestamp;
  63     }
  64   }
  65
  66   class KeepAllDeletionPolicy implements IndexDeletionPolicy {
  67     int numOnInit;
  68     int numOnCommit;
  69     Directory dir;
  70     public void onInit(List<? extends IndexCommit> commits) throws IOException {
  71       verifyCommitOrder(commits);
  72       numOnInit++;
  73     }
  74     public void onCommit(List<? extends IndexCommit> commits) throws IOException {
  75       IndexCommit lastCommit =  commits.get(commits.size()-1);
  76       IndexReader r = IndexReader.open(dir, true);
  77       assertEquals("lastCommit.segmentCount()=" + lastCommit.getSegmentCount() + " vs IndexReader.segmentCount=" + r.getSequentialSubReaders().length, r.getSequentialSubReaders().length, lastCommit.getSegmentCount());
  78       r.close();
  79       verifyCommitOrder(commits);
  80       numOnCommit++;
  81     }
  82   }
  83
  84   /**
  85    * This is useful for adding to a big index when you know
  86    * readers are not using it.
  87    */
  88   class KeepNoneOnInitDeletionPolicy implements IndexDeletionPolicy {
  89     int numOnInit;
  90     int numOnCommit;
  91     public void onInit(List<? extends IndexCommit> commits) throws IOException {
  92       verifyCommitOrder(commits);
  93       numOnInit++;
  94       // On init, delete all commit points:
  95       for (final IndexCommit commit : commits) {
  96         commit.delete();
  97         assertTrue(commit.isDeleted());
  98       }
  99     }
 100     public void onCommit(List<? extends IndexCommit> commits) throws IOException {
 101       verifyCommitOrder(commits);
 102       int size = commits.size();
 103       // Delete all but last one:
 104       for(int i=0;i<size-1;i++) {
 105         ((IndexCommit) commits.get(i)).delete();
 106       }
 107       numOnCommit++;
 108     }
 109   }
 110
 111   class KeepLastNDeletionPolicy implements IndexDeletionPolicy {
 112     int numOnInit;
 113     int numOnCommit;
 114     int numToKeep;
 115     int numDelete;
 116     Set<String> seen = new HashSet<String>();
 117
 118     public KeepLastNDeletionPolicy(int numToKeep) {
 119       this.numToKeep = numToKeep;
 120     }
 121
 122     public void onInit(List<? extends IndexCommit> commits) throws IOException {
 123       if (VERBOSE) {
 124         System.out.println("TEST: onInit");
 125       }
 126       verifyCommitOrder(commits);
 127       numOnInit++;
 128       // do no deletions on init
 129       doDeletes(commits, false);
 130     }
 131
 132     public void onCommit(List<? extends IndexCommit> commits) throws IOException {
 133       if (VERBOSE) {
 134         System.out.println("TEST: onCommit");
 135       }
 136       verifyCommitOrder(commits);
 137       doDeletes(commits, true);
 138     }
 139
 140     private void doDeletes(List<? extends IndexCommit> commits, boolean isCommit) {
 141
 142       // Assert that we really are only called for each new
 143       // commit:
 144       if (isCommit) {
 145         String fileName = ((IndexCommit) commits.get(commits.size()-1)).getSegmentsFileName();
 146         if (seen.contains(fileName)) {
 147           throw new RuntimeException("onCommit was called twice on the same commit point: " + fileName);
 148         }
 149         seen.add(fileName);
 150         numOnCommit++;
 151       }
 152       int size = commits.size();
 153       for(int i=0;i<size-numToKeep;i++) {
 154         ((IndexCommit) commits.get(i)).delete();
 155         numDelete++;
 156       }
 157     }
 158   }
 159
 160   /*
 161    * Delete a commit only when it has been obsoleted by N
 162    * seconds.
 163    */
 164   class ExpirationTimeDeletionPolicy implements IndexDeletionPolicy {
 165
 166     Directory dir;
 167     double expirationTimeSeconds;
 168     int numDelete;
 169
 170     public ExpirationTimeDeletionPolicy(Directory dir, double seconds) {
 171       this.dir = dir;
 172       this.expirationTimeSeconds = seconds;
 173     }
 174
 175     public void onInit(List<? extends IndexCommit> commits) throws IOException {
 176       verifyCommitOrder(commits);
 177       onCommit(commits);
 178     }
 179
 180     public void onCommit(List<? extends IndexCommit> commits) throws IOException {
 181       verifyCommitOrder(commits);
 182
 183       IndexCommit lastCommit = commits.get(commits.size()-1);
 184
 185       // Any commit older than expireTime should be deleted:
 186       double expireTime = dir.fileModified(lastCommit.getSegmentsFileName())/1000.0 - expirationTimeSeconds;
 187
 188       for (final IndexCommit commit : commits) {
 189         double modTime = dir.fileModified(commit.getSegmentsFileName())/1000.0;
 190         if (commit != lastCommit && modTime < expireTime) {
 191           commit.delete();
 192           numDelete += 1;
 193         }
 194       }
 195     }
 196   }
 197
 198   /*
 199    * Test "by time expiration" deletion policy:
 200    */
 201   public void testExpirationTimeDeletionPolicy() throws IOException, InterruptedException {
 202
 203     final double SECONDS = 2.0;
 204
 205     Directory dir = newDirectory();
 206     ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS);
 207     IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
 208         new MockAnalyzer(random))
 209         .setIndexDeletionPolicy(policy);
 210     MergePolicy mp = conf.getMergePolicy();
 211     if (mp instanceof LogMergePolicy) {
 212       setUseCompoundFile(mp, true);
 213     }
 214     IndexWriter writer = new IndexWriter(dir, conf);
 215     writer.close();
 216
 217     final int ITER = 9;
 218
 219     long lastDeleteTime = 0;
 220     for(int i=0;i<ITER;i++) {
 221       // Record last time when writer performed deletes of
 222       // past commits
 223       lastDeleteTime = System.currentTimeMillis();
 224       conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
 225           new MockAnalyzer(random)).setOpenMode(
 226           OpenMode.APPEND).setIndexDeletionPolicy(policy);
 227       mp = conf.getMergePolicy();
 228       if (mp instanceof LogMergePolicy) {
 229         setUseCompoundFile(mp, true);
 230       }
 231       writer = new IndexWriter(dir, conf);
 232       for(int j=0;j<17;j++) {
 233         addDoc(writer);
 234       }
 235       writer.close();
 236
 237       if (i < ITER-1) {
 238         // Make sure to sleep long enough so that some commit
 239         // points will be deleted:
 240         Thread.sleep((int) (1000.0*(SECONDS/5.0)));
 241       }
 242     }
 243
 244     // First, make sure the policy in fact deleted something:
 245     assertTrue("no commits were deleted", policy.numDelete > 0);
 246
 247     // Then simplistic check: just verify that the
 248     // segments_N's that still exist are in fact within SECONDS
 249     // seconds of the last one's mod time, and, that I can
 250     // open a reader on each:
 251     long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 252
 253     String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
 254                                                             "",
 255                                                             gen);
 256     dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 257
 258     boolean oneSecondResolution = true;
 259
 260     while(gen > 0) {
 261       try {
 262         IndexReader reader = IndexReader.open(dir, true);
 263         reader.close();
 264         fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
 265                                                          "",
 266                                                          gen);
 267
 268         // if we are on a filesystem that seems to have only
 269         // 1 second resolution, allow +1 second in commit
 270         // age tolerance:
 271         long modTime = dir.fileModified(fileName);
 272         oneSecondResolution &= (modTime % 1000) == 0;
 273         final long leeway = (long) ((SECONDS + (oneSecondResolution ? 1.0:0.0))*1000);
 274
 275         assertTrue("commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted ", lastDeleteTime - modTime <= leeway);
 276       } catch (IOException e) {
 277         // OK
 278         break;
 279       }
 280
 281       dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 282       gen--;
 283     }
 284
 285     dir.close();
 286   }
 287
 288   /*
 289    * Test a silly deletion policy that keeps all commits around.
 290    */
 291   public void testKeepAllDeletionPolicy() throws IOException {
 292     for(int pass=0;pass<2;pass++) {
 293
 294       if (VERBOSE) {
 295         System.out.println("TEST: cycle pass=" + pass);
 296       }
 297
 298       boolean useCompoundFile = (pass % 2) != 0;
 299
 300       // Never deletes a commit
 301       KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();
 302
 303       Directory dir = newDirectory();
 304       policy.dir = dir;
 305
 306       IndexWriterConfig conf = newIndexWriterConfig(
 307           TEST_VERSION_CURRENT, new MockAnalyzer(random))
 308           .setIndexDeletionPolicy(policy).setMaxBufferedDocs(10)
 309           .setMergeScheduler(new SerialMergeScheduler());
 310       MergePolicy mp = conf.getMergePolicy();
 311       if (mp instanceof LogMergePolicy) {
 312         setUseCompoundFile(mp, useCompoundFile);
 313       }
 314       IndexWriter writer = new IndexWriter(dir, conf);
 315       for(int i=0;i<107;i++) {
 316         addDoc(writer);
 317       }
 318       writer.close();
 319
 320       final boolean needsMerging;
 321       {
 322         IndexReader r = IndexReader.open(dir);
 323         needsMerging = r.getSequentialSubReaders().length != 1;
 324         r.close();
 325       }
 326       if (needsMerging) {
 327         conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
 328                                     new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(
 329                                                                                               OpenMode.APPEND).setIndexDeletionPolicy(policy);
 330         mp = conf.getMergePolicy();
 331         if (mp instanceof LogMergePolicy) {
 332           setUseCompoundFile(mp, true);
 333         }
 334         if (VERBOSE) {
 335           System.out.println("TEST: open writer for forceMerge");
 336         }
 337         writer = new IndexWriter(dir, conf);
 338         writer.setInfoStream(VERBOSE ? System.out : null);
 339         writer.forceMerge(1);
 340         writer.close();
 341       }
 342       assertEquals(needsMerging ? 1:0, policy.numOnInit);
 343
 344       // If we are not auto committing then there should
 345       // be exactly 2 commits (one per close above):
 346       assertEquals(1 + (needsMerging ? 1:0), policy.numOnCommit);
 347
 348       // Test listCommits
 349       Collection<IndexCommit> commits = IndexReader.listCommits(dir);
 350       // 2 from closing writer
 351       assertEquals(1 + (needsMerging ? 1:0), commits.size());
 352
 353       // Make sure we can open a reader on each commit:
 354       for (final IndexCommit commit : commits) {
 355         IndexReader r = IndexReader.open(commit, null, false);
 356         r.close();
 357       }
 358
 359       // Simplistic check: just verify all segments_N's still
 360       // exist, and, I can open a reader on each:
 361       dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 362       long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 363       while(gen > 0) {
 364         IndexReader reader = IndexReader.open(dir, true);
 365         reader.close();
 366         dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 367         gen--;
 368
 369         if (gen > 0) {
 370           // Now that we've removed a commit point, which
 371           // should have orphan'd at least one index file.
 372           // Open & close a writer and assert that it
 373           // actually removed something:
 374           int preCount = dir.listAll().length;
 375           writer = new IndexWriter(dir, newIndexWriterConfig(
 376               TEST_VERSION_CURRENT,
 377               new MockAnalyzer(random)).setOpenMode(
 378               OpenMode.APPEND).setIndexDeletionPolicy(policy));
 379           writer.close();
 380           int postCount = dir.listAll().length;
 381           assertTrue(postCount < preCount);
 382         }
 383       }
 384
 385       dir.close();
 386     }
 387   }
 388
 389   /* Uses KeepAllDeletionPolicy to keep all commits around,
 390    * then, opens a new IndexWriter on a previous commit
 391    * point. */
 392   public void testOpenPriorSnapshot() throws IOException {
 393     // Never deletes a commit
 394     KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();
 395
 396     Directory dir = newDirectory();
 397     policy.dir = dir;
 398
 399     IndexWriter writer = new IndexWriter(
 400         dir,
 401         newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
 402             setIndexDeletionPolicy(policy).
 403             setMaxBufferedDocs(2).
 404             setMergePolicy(newLogMergePolicy(10))
 405     );
 406     for(int i=0;i<10;i++) {
 407       addDoc(writer);
 408       if ((1+i)%2 == 0)
 409         writer.commit();
 410     }
 411     writer.close();
 412
 413     Collection<IndexCommit> commits = IndexReader.listCommits(dir);
 414     assertEquals(5, commits.size());
 415     IndexCommit lastCommit = null;
 416     for (final IndexCommit commit : commits) {
 417       if (lastCommit == null || commit.getGeneration() > lastCommit.getGeneration())
 418         lastCommit = commit;
 419     }
 420     assertTrue(lastCommit != null);
 421
 422     // Now add 1 doc and merge
 423     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
 424         new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
 425     addDoc(writer);
 426     assertEquals(11, writer.numDocs());
 427     writer.forceMerge(1);
 428     writer.close();
 429
 430     assertEquals(6, IndexReader.listCommits(dir).size());
 431
 432     // Now open writer on the commit just before merge:
 433     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
 434         .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
 435     assertEquals(10, writer.numDocs());
 436
 437     // Should undo our rollback:
 438     writer.rollback();
 439
 440     IndexReader r = IndexReader.open(dir, true);
 441     // Still merged, still 11 docs
 442     assertEquals(1, r.getSequentialSubReaders().length);
 443     assertEquals(11, r.numDocs());
 444     r.close();
 445
 446     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
 447         .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
 448     assertEquals(10, writer.numDocs());
 449     // Commits the rollback:
 450     writer.close();
 451
 452     // Now 8 because we made another commit
 453     assertEquals(7, IndexReader.listCommits(dir).size());
 454
 455     r = IndexReader.open(dir, true);
 456     // Not fully merged because we rolled it back, and now only
 457     // 10 docs
 458     assertTrue(r.getSequentialSubReaders().length > 1);
 459     assertEquals(10, r.numDocs());
 460     r.close();
 461
 462     // Re-merge
 463     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
 464     writer.forceMerge(1);
 465     writer.close();
 466
 467     r = IndexReader.open(dir, true);
 468     assertEquals(1, r.getSequentialSubReaders().length);
 469     assertEquals(10, r.numDocs());
 470     r.close();
 471
 472     // Now open writer on the commit just before merging,
 473     // but this time keeping only the last commit:
 474     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexCommit(lastCommit));
 475     assertEquals(10, writer.numDocs());
 476
 477     // Reader still sees fully merged index, because writer
 478     // opened on the prior commit has not yet committed:
 479     r = IndexReader.open(dir, true);
 480     assertEquals(1, r.getSequentialSubReaders().length);
 481     assertEquals(10, r.numDocs());
 482     r.close();
 483
 484     writer.close();
 485
 486     // Now reader sees not-fully-merged index:
 487     r = IndexReader.open(dir, true);
 488     assertTrue(r.getSequentialSubReaders().length > 1);
 489     assertEquals(10, r.numDocs());
 490     r.close();
 491
 492     dir.close();
 493   }
 494
 495
 496   /* Test keeping NO commit points.  This is a viable and
 497    * useful case eg where you want to build a big index and
 498    * you know there are no readers.
 499    */
 500   public void testKeepNoneOnInitDeletionPolicy() throws IOException {
 501     for(int pass=0;pass<2;pass++) {
 502
 503       boolean useCompoundFile = (pass % 2) != 0;
 504
 505       KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy();
 506
 507       Directory dir = newDirectory();
 508
 509       IndexWriterConfig conf = newIndexWriterConfig(
 510           TEST_VERSION_CURRENT, new MockAnalyzer(random))
 511           .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
 512           .setMaxBufferedDocs(10);
 513       MergePolicy mp = conf.getMergePolicy();
 514       if (mp instanceof LogMergePolicy) {
 515         setUseCompoundFile(mp, useCompoundFile);
 516       }
 517       IndexWriter writer = new IndexWriter(dir, conf);
 518       for(int i=0;i<107;i++) {
 519         addDoc(writer);
 520       }
 521       writer.close();
 522
 523       conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
 524           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
 525       mp = conf.getMergePolicy();
 526       if (mp instanceof LogMergePolicy) {
 527         setUseCompoundFile(mp, true);
 528       }
 529       writer = new IndexWriter(dir, conf);
 530       writer.forceMerge(1);
 531       writer.close();
 532
 533       assertEquals(1, policy.numOnInit);
 534       // If we are not auto committing then there should
 535       // be exactly 2 commits (one per close above):
 536       assertEquals(2, policy.numOnCommit);
 537
 538       // Simplistic check: just verify the index is in fact
 539       // readable:
 540       IndexReader reader = IndexReader.open(dir, true);
 541       reader.close();
 542
 543       dir.close();
 544     }
 545   }
 546
 547   /*
 548    * Test a deletion policy that keeps last N commits.
 549    */
 550   public void testKeepLastNDeletionPolicy() throws IOException {
 551     final int N = 5;
 552
 553     for(int pass=0;pass<2;pass++) {
 554
 555       boolean useCompoundFile = (pass % 2) != 0;
 556
 557       Directory dir = newDirectory();
 558
 559       KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
 560
 561       for(int j=0;j<N+1;j++) {
 562         IndexWriterConfig conf = newIndexWriterConfig(
 563             TEST_VERSION_CURRENT, new MockAnalyzer(random))
 564             .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
 565             .setMaxBufferedDocs(10);
 566         MergePolicy mp = conf.getMergePolicy();
 567         if (mp instanceof LogMergePolicy) {
 568           setUseCompoundFile(mp, useCompoundFile);
 569         }
 570         IndexWriter writer = new IndexWriter(dir, conf);
 571         for(int i=0;i<17;i++) {
 572           addDoc(writer);
 573         }
 574         writer.forceMerge(1);
 575         writer.close();
 576       }
 577
 578       assertTrue(policy.numDelete > 0);
 579       assertEquals(N, policy.numOnInit);
 580       assertEquals(N+1, policy.numOnCommit);
 581
 582       // Simplistic check: just verify only the past N segments_N's still
 583       // exist, and, I can open a reader on each:
 584       dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 585       long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 586       for(int i=0;i<N+1;i++) {
 587         try {
 588           IndexReader reader = IndexReader.open(dir, true);
 589           reader.close();
 590           if (i == N) {
 591             fail("should have failed on commits prior to last " + N);
 592           }
 593         } catch (IOException e) {
 594           if (i != N) {
 595             throw e;
 596           }
 597         }
 598         if (i < N) {
 599           dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 600         }
 601         gen--;
 602       }
 603
 604       dir.close();
 605     }
 606   }
 607
 608   /*
 609    * Test a deletion policy that keeps last N commits
 610    * around, with reader doing deletes.
 611    */
 612   public void testKeepLastNDeletionPolicyWithReader() throws IOException {
 613     final int N = 10;
 614
 615     for(int pass=0;pass<2;pass++) {
 616
 617       boolean useCompoundFile = (pass % 2) != 0;
 618
 619       KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
 620
 621       Directory dir = newDirectory();
 622       IndexWriterConfig conf = newIndexWriterConfig(
 623           TEST_VERSION_CURRENT, new MockAnalyzer(random))
 624         .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
 625       MergePolicy mp = conf.getMergePolicy();
 626       if (mp instanceof LogMergePolicy) {
 627         setUseCompoundFile(mp, useCompoundFile);
 628       }
 629       IndexWriter writer = new IndexWriter(dir, conf);
 630       writer.close();
 631       Term searchTerm = new Term("content", "aaa");
 632       Query query = new TermQuery(searchTerm);
 633
 634       for(int i=0;i<N+1;i++) {
 635         if (VERBOSE) {
 636           System.out.println("\nTEST: cycle i=" + i);
 637         }
 638         conf = newIndexWriterConfig(
 639             TEST_VERSION_CURRENT, new MockAnalyzer(random))
 640           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
 641         mp = conf.getMergePolicy();
 642         if (mp instanceof LogMergePolicy) {
 643           setUseCompoundFile(mp, useCompoundFile);
 644         }
 645         writer = new IndexWriter(dir, conf);
 646         writer.setInfoStream(VERBOSE ? System.out : null);
 647         for(int j=0;j<17;j++) {
 648           addDoc(writer);
 649         }
 650         // this is a commit
 651         if (VERBOSE) {
 652           System.out.println("TEST: close writer");
 653         }
 654         writer.close();
 655         IndexReader reader = IndexReader.open(dir, policy, false);
 656         reader.deleteDocument(3*i+1);
 657         reader.setNorm(4*i+1, "content", 2.0F);
 658         IndexSearcher searcher = newSearcher(reader);
 659         ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
 660         assertEquals(16*(1+i), hits.length);
 661         // this is a commit
 662         if (VERBOSE) {
 663           System.out.println("TEST: close reader numOnCommit=" + policy.numOnCommit);
 664         }
 665         reader.close();
 666         searcher.close();
 667       }
 668       conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
 669           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
 670       mp = conf.getMergePolicy();
 671       if (mp instanceof LogMergePolicy) {
 672         setUseCompoundFile(mp, useCompoundFile);
 673       }
 674       IndexReader r = IndexReader.open(dir);
 675       final boolean wasFullyMerged = r.getSequentialSubReaders().length == 1 && !r.hasDeletions();
 676       r.close();
 677       writer = new IndexWriter(dir, conf);
 678       writer.forceMerge(1);
 679       // this is a commit
 680       writer.close();
 681
 682       assertEquals(2*(N+1)+1, policy.numOnInit);
 683       assertEquals(2*(N+2) - (wasFullyMerged ? 1:0), policy.numOnCommit);
 684
 685       IndexReader rwReader = IndexReader.open(dir, false);
 686       IndexSearcher searcher = new IndexSearcher(rwReader);
 687       ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
 688       assertEquals(176, hits.length);
 689
 690       // Simplistic check: just verify only the past N segments_N's still
 691       // exist, and, I can open a reader on each:
 692       long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 693
 694       dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 695       int expectedCount = 176;
 696       searcher.close();
 697       rwReader.close();
 698       for(int i=0;i<N+1;i++) {
 699         try {
 700           IndexReader reader = IndexReader.open(dir, true);
 701
 702           // Work backwards in commits on what the expected
 703           // count should be.
 704           searcher = newSearcher(reader);
 705           hits = searcher.search(query, null, 1000).scoreDocs;
 706           if (i > 1) {
 707             if (i % 2 == 0) {
 708               expectedCount += 1;
 709             } else {
 710               expectedCount -= 17;
 711             }
 712           }
 713           assertEquals(expectedCount, hits.length);
 714           searcher.close();
 715           reader.close();
 716           if (i == N) {
 717             fail("should have failed on commits before last 5");
 718           }
 719         } catch (IOException e) {
 720           if (i != N) {
 721             throw e;
 722           }
 723         }
 724         if (i < N) {
 725           dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 726         }
 727         gen--;
 728       }
 729       dir.close();
 730     }
 731   }
 732
 733   /*
 734    * Test a deletion policy that keeps last N commits
 735    * around, through creates.
 736    */
 737   public void testKeepLastNDeletionPolicyWithCreates() throws IOException {
 738
 739     final int N = 10;
 740
 741     for(int pass=0;pass<2;pass++) {
 742
 743       boolean useCompoundFile = (pass % 2) != 0;
 744
 745       KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
 746
 747       Directory dir = newDirectory();
 748       IndexWriterConfig conf = newIndexWriterConfig(
 749           TEST_VERSION_CURRENT, new MockAnalyzer(random))
 750           .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
 751           .setMaxBufferedDocs(10);
 752       MergePolicy mp = conf.getMergePolicy();
 753       if (mp instanceof LogMergePolicy) {
 754         setUseCompoundFile(mp, useCompoundFile);
 755       }
 756       IndexWriter writer = new IndexWriter(dir, conf);
 757       writer.close();
 758       Term searchTerm = new Term("content", "aaa");
 759       Query query = new TermQuery(searchTerm);
 760
 761       for(int i=0;i<N+1;i++) {
 762
 763         conf = newIndexWriterConfig(
 764             TEST_VERSION_CURRENT, new MockAnalyzer(random))
 765             .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy)
 766             .setMaxBufferedDocs(10);
 767         mp = conf.getMergePolicy();
 768         if (mp instanceof LogMergePolicy) {
 769           setUseCompoundFile(mp, useCompoundFile);
 770         }
 771         writer = new IndexWriter(dir, conf);
 772         for(int j=0;j<17;j++) {
 773           addDoc(writer);
 774         }
 775         // this is a commit
 776         writer.close();
 777         IndexReader reader = IndexReader.open(dir, policy, false);
 778         reader.deleteDocument(3);
 779         reader.setNorm(5, "content", 2.0F);
 780         IndexSearcher searcher = newSearcher(reader);
 781         ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
 782         assertEquals(16, hits.length);
 783         // this is a commit
 784         reader.close();
 785         searcher.close();
 786
 787         writer = new IndexWriter(dir, newIndexWriterConfig(
 788             TEST_VERSION_CURRENT, new MockAnalyzer(random))
 789             .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy));
 790         // This will not commit: there are no changes
 791         // pending because we opened for "create":
 792         writer.close();
 793       }
 794
 795       assertEquals(3*(N+1), policy.numOnInit);
 796       assertEquals(3*(N+1)+1, policy.numOnCommit);
 797
 798       IndexReader rwReader = IndexReader.open(dir, false);
 799       IndexSearcher searcher = new IndexSearcher(rwReader);
 800       ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
 801       assertEquals(0, hits.length);
 802
 803       // Simplistic check: just verify only the past N segments_N's still
 804       // exist, and, I can open a reader on each:
 805       long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 806
 807       dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 808       int expectedCount = 0;
 809
 810       searcher.close();
 811       rwReader.close();
 812
 813       for(int i=0;i<N+1;i++) {
 814         try {
 815           IndexReader reader = IndexReader.open(dir, true);
 816
 817           // Work backwards in commits on what the expected
 818           // count should be.
 819           searcher = newSearcher(reader);
 820           hits = searcher.search(query, null, 1000).scoreDocs;
 821           assertEquals(expectedCount, hits.length);
 822           searcher.close();
 823           if (expectedCount == 0) {
 824             expectedCount = 16;
 825           } else if (expectedCount == 16) {
 826             expectedCount = 17;
 827           } else if (expectedCount == 17) {
 828             expectedCount = 0;
 829           }
 830           reader.close();
 831           if (i == N) {
 832             fail("should have failed on commits before last " + N);
 833           }
 834         } catch (IOException e) {
 835           if (i != N) {
 836             throw e;
 837           }
 838         }
 839         if (i < N) {
 840           dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 841         }
 842         gen--;
 843       }
 844
 845       dir.close();
 846     }
 847   }
 848
 849   private void addDoc(IndexWriter writer) throws IOException
 850   {
 851     Document doc = new Document();
 852     doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
 853     writer.addDocument(doc);
 854   }
 855 }