lucene-java-3.5.0/lucene/backwards/src/test/org/apache/lucene/index/TestDeletionPolicy.java

   1 package org.apache.lucene.index;
   2
   3 /**
   4  * Licensed to the Apache Software Foundation (ASF) under one or more
   5  * contributor license agreements.  See the NOTICE file distributed with
   6  * this work for additional information regarding copyright ownership.
   7  * The ASF licenses this file to You under the Apache License, Version 2.0
   8  * (the "License"); you may not use this file except in compliance with
   9  * the License.  You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19
  20 import java.io.IOException;
  21 import java.util.HashSet;
  22 import java.util.List;
  23 import java.util.Set;
  24 import java.util.Collection;
  25
  26 import org.apache.lucene.analysis.MockAnalyzer;
  27 import org.apache.lucene.analysis.WhitespaceAnalyzer;
  28 import org.apache.lucene.document.Document;
  29 import org.apache.lucene.document.Field;
  30 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
  31 import org.apache.lucene.search.IndexSearcher;
  32 import org.apache.lucene.search.Query;
  33 import org.apache.lucene.search.ScoreDoc;
  34 import org.apache.lucene.search.TermQuery;
  35 import org.apache.lucene.store.Directory;
  36 import org.apache.lucene.util.LuceneTestCase;
  37
  38 /*
  39   Verify we can read the pre-2.1 file format, do searches
  40   against it, and add documents to it.
  41 */
  42
  43 public class TestDeletionPolicy extends LuceneTestCase {
  44
  45   private void verifyCommitOrder(List<? extends IndexCommit> commits) throws IOException {
  46     final IndexCommit firstCommit =  commits.get(0);
  47     long last = SegmentInfos.generationFromSegmentsFileName(firstCommit.getSegmentsFileName());
  48     assertEquals(last, firstCommit.getGeneration());
  49     long lastVersion = firstCommit.getVersion();
  50     long lastTimestamp = firstCommit.getTimestamp();
  51     for(int i=1;i<commits.size();i++) {
  52       final IndexCommit commit =  commits.get(i);
  53       long now = SegmentInfos.generationFromSegmentsFileName(commit.getSegmentsFileName());
  54       long nowVersion = commit.getVersion();
  55       long nowTimestamp = commit.getTimestamp();
  56       assertTrue("SegmentInfos commits are out-of-order", now > last);
  57       assertTrue("SegmentInfos versions are out-of-order", nowVersion > lastVersion);
  58       assertTrue("SegmentInfos timestamps are out-of-order: now=" + nowTimestamp + " vs last=" + lastTimestamp, nowTimestamp >= lastTimestamp);
  59       assertEquals(now, commit.getGeneration());
  60       last = now;
  61       lastVersion = nowVersion;
  62       lastTimestamp = nowTimestamp;
  63     }
  64   }
  65
  66   class KeepAllDeletionPolicy implements IndexDeletionPolicy {
  67     int numOnInit;
  68     int numOnCommit;
  69     Directory dir;
  70     public void onInit(List<? extends IndexCommit> commits) throws IOException {
  71       verifyCommitOrder(commits);
  72       numOnInit++;
  73     }
  74     public void onCommit(List<? extends IndexCommit> commits) throws IOException {
  75       IndexCommit lastCommit =  commits.get(commits.size()-1);
  76       IndexReader r = IndexReader.open(dir, true);
  77       r.close();
  78       verifyCommitOrder(commits);
  79       numOnCommit++;
  80     }
  81   }
  82
  83   /**
  84    * This is useful for adding to a big index when you know
  85    * readers are not using it.
  86    */
  87   class KeepNoneOnInitDeletionPolicy implements IndexDeletionPolicy {
  88     int numOnInit;
  89     int numOnCommit;
  90     public void onInit(List<? extends IndexCommit> commits) throws IOException {
  91       verifyCommitOrder(commits);
  92       numOnInit++;
  93       // On init, delete all commit points:
  94       for (final IndexCommit commit : commits) {
  95         commit.delete();
  96         assertTrue(commit.isDeleted());
  97       }
  98     }
  99     public void onCommit(List<? extends IndexCommit> commits) throws IOException {
 100       verifyCommitOrder(commits);
 101       int size = commits.size();
 102       // Delete all but last one:
 103       for(int i=0;i<size-1;i++) {
 104         ((IndexCommit) commits.get(i)).delete();
 105       }
 106       numOnCommit++;
 107     }
 108   }
 109
 110   class KeepLastNDeletionPolicy implements IndexDeletionPolicy {
 111     int numOnInit;
 112     int numOnCommit;
 113     int numToKeep;
 114     int numDelete;
 115     Set<String> seen = new HashSet<String>();
 116
 117     public KeepLastNDeletionPolicy(int numToKeep) {
 118       this.numToKeep = numToKeep;
 119     }
 120
 121     public void onInit(List<? extends IndexCommit> commits) throws IOException {
 122       if (VERBOSE) {
 123         System.out.println("TEST: onInit");
 124       }
 125       verifyCommitOrder(commits);
 126       numOnInit++;
 127       // do no deletions on init
 128       doDeletes(commits, false);
 129     }
 130
 131     public void onCommit(List<? extends IndexCommit> commits) throws IOException {
 132       if (VERBOSE) {
 133         System.out.println("TEST: onCommit");
 134       }
 135       verifyCommitOrder(commits);
 136       doDeletes(commits, true);
 137     }
 138
 139     private void doDeletes(List<? extends IndexCommit> commits, boolean isCommit) {
 140
 141       // Assert that we really are only called for each new
 142       // commit:
 143       if (isCommit) {
 144         String fileName = ((IndexCommit) commits.get(commits.size()-1)).getSegmentsFileName();
 145         if (seen.contains(fileName)) {
 146           throw new RuntimeException("onCommit was called twice on the same commit point: " + fileName);
 147         }
 148         seen.add(fileName);
 149         numOnCommit++;
 150       }
 151       int size = commits.size();
 152       for(int i=0;i<size-numToKeep;i++) {
 153         ((IndexCommit) commits.get(i)).delete();
 154         numDelete++;
 155       }
 156     }
 157   }
 158
 159   /*
 160    * Delete a commit only when it has been obsoleted by N
 161    * seconds.
 162    */
 163   class ExpirationTimeDeletionPolicy implements IndexDeletionPolicy {
 164
 165     Directory dir;
 166     double expirationTimeSeconds;
 167     int numDelete;
 168
 169     public ExpirationTimeDeletionPolicy(Directory dir, double seconds) {
 170       this.dir = dir;
 171       this.expirationTimeSeconds = seconds;
 172     }
 173
 174     public void onInit(List<? extends IndexCommit> commits) throws IOException {
 175       verifyCommitOrder(commits);
 176       onCommit(commits);
 177     }
 178
 179     public void onCommit(List<? extends IndexCommit> commits) throws IOException {
 180       verifyCommitOrder(commits);
 181
 182       IndexCommit lastCommit = commits.get(commits.size()-1);
 183
 184       // Any commit older than expireTime should be deleted:
 185       double expireTime = dir.fileModified(lastCommit.getSegmentsFileName())/1000.0 - expirationTimeSeconds;
 186
 187       for (final IndexCommit commit : commits) {
 188         double modTime = dir.fileModified(commit.getSegmentsFileName())/1000.0;
 189         if (commit != lastCommit && modTime < expireTime) {
 190           commit.delete();
 191           numDelete += 1;
 192         }
 193       }
 194     }
 195   }
 196
 197   /*
 198    * Test "by time expiration" deletion policy:
 199    */
 200   public void testExpirationTimeDeletionPolicy() throws IOException, InterruptedException {
 201
 202     final double SECONDS = 2.0;
 203
 204     Directory dir = newDirectory();
 205     ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS);
 206     IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
 207         new MockAnalyzer(random))
 208         .setIndexDeletionPolicy(policy);
 209     MergePolicy mp = conf.getMergePolicy();
 210     if (mp instanceof LogMergePolicy) {
 211       setUseCompoundFile(mp, true);
 212     }
 213     IndexWriter writer = new IndexWriter(dir, conf);
 214     writer.close();
 215
 216     final int ITER = 9;
 217
 218     long lastDeleteTime = 0;
 219     for(int i=0;i<ITER;i++) {
 220       // Record last time when writer performed deletes of
 221       // past commits
 222       lastDeleteTime = System.currentTimeMillis();
 223       conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
 224           new MockAnalyzer(random)).setOpenMode(
 225           OpenMode.APPEND).setIndexDeletionPolicy(policy);
 226       mp = conf.getMergePolicy();
 227       if (mp instanceof LogMergePolicy) {
 228         setUseCompoundFile(mp, true);
 229       }
 230       writer = new IndexWriter(dir, conf);
 231       for(int j=0;j<17;j++) {
 232         addDoc(writer);
 233       }
 234       writer.close();
 235
 236       if (i < ITER-1) {
 237         // Make sure to sleep long enough so that some commit
 238         // points will be deleted:
 239         Thread.sleep((int) (1000.0*(SECONDS/5.0)));
 240       }
 241     }
 242
 243     // First, make sure the policy in fact deleted something:
 244     assertTrue("no commits were deleted", policy.numDelete > 0);
 245
 246     // Then simplistic check: just verify that the
 247     // segments_N's that still exist are in fact within SECONDS
 248     // seconds of the last one's mod time, and, that I can
 249     // open a reader on each:
 250     long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 251
 252     String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
 253                                                             "",
 254                                                             gen);
 255     dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 256
 257     boolean oneSecondResolution = true;
 258
 259     while(gen > 0) {
 260       try {
 261         IndexReader reader = IndexReader.open(dir, true);
 262         reader.close();
 263         fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
 264                                                          "",
 265                                                          gen);
 266
 267         // if we are on a filesystem that seems to have only
 268         // 1 second resolution, allow +1 second in commit
 269         // age tolerance:
 270         long modTime = dir.fileModified(fileName);
 271         oneSecondResolution &= (modTime % 1000) == 0;
 272         final long leeway = (long) ((SECONDS + (oneSecondResolution ? 1.0:0.0))*1000);
 273
 274         assertTrue("commit point was older than " + SECONDS + " seconds (" + (lastDeleteTime - modTime) + " msec) but did not get deleted ", lastDeleteTime - modTime <= leeway);
 275       } catch (IOException e) {
 276         // OK
 277         break;
 278       }
 279
 280       dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 281       gen--;
 282     }
 283
 284     dir.close();
 285   }
 286
 287   /*
 288    * Test a silly deletion policy that keeps all commits around.
 289    */
 290   public void testKeepAllDeletionPolicy() throws IOException {
 291     for(int pass=0;pass<2;pass++) {
 292
 293       if (VERBOSE) {
 294         System.out.println("TEST: cycle pass=" + pass);
 295       }
 296
 297       boolean useCompoundFile = (pass % 2) != 0;
 298
 299       // Never deletes a commit
 300       KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();
 301
 302       Directory dir = newDirectory();
 303       policy.dir = dir;
 304
 305       IndexWriterConfig conf = newIndexWriterConfig(
 306           TEST_VERSION_CURRENT, new MockAnalyzer(random))
 307           .setIndexDeletionPolicy(policy).setMaxBufferedDocs(10)
 308           .setMergeScheduler(new SerialMergeScheduler());
 309       MergePolicy mp = conf.getMergePolicy();
 310       if (mp instanceof LogMergePolicy) {
 311         setUseCompoundFile(mp, useCompoundFile);
 312       }
 313       IndexWriter writer = new IndexWriter(dir, conf);
 314       for(int i=0;i<107;i++) {
 315         addDoc(writer);
 316       }
 317       writer.close();
 318
 319       final boolean isOptimized;
 320       {
 321         IndexReader r = IndexReader.open(dir);
 322         isOptimized = r.isOptimized();
 323         r.close();
 324       }
 325       if (!isOptimized) {
 326         conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
 327                                     new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(
 328                                                                                               OpenMode.APPEND).setIndexDeletionPolicy(policy);
 329         mp = conf.getMergePolicy();
 330         if (mp instanceof LogMergePolicy) {
 331           setUseCompoundFile(mp, true);
 332         }
 333         if (VERBOSE) {
 334           System.out.println("TEST: open writer for optimize");
 335         }
 336         writer = new IndexWriter(dir, conf);
 337         writer.setInfoStream(VERBOSE ? System.out : null);
 338         writer.optimize();
 339         writer.close();
 340       }
 341       assertEquals(isOptimized ? 0:1, policy.numOnInit);
 342
 343       // If we are not auto committing then there should
 344       // be exactly 2 commits (one per close above):
 345       assertEquals(1 + (isOptimized ? 0:1), policy.numOnCommit);
 346
 347       // Test listCommits
 348       Collection<IndexCommit> commits = IndexReader.listCommits(dir);
 349       // 2 from closing writer
 350       assertEquals(1 + (isOptimized ? 0:1), commits.size());
 351
 352       // Make sure we can open a reader on each commit:
 353       for (final IndexCommit commit : commits) {
 354         IndexReader r = IndexReader.open(commit, null, false);
 355         r.close();
 356       }
 357
 358       // Simplistic check: just verify all segments_N's still
 359       // exist, and, I can open a reader on each:
 360       dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 361       long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 362       while(gen > 0) {
 363         IndexReader reader = IndexReader.open(dir, true);
 364         reader.close();
 365         dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 366         gen--;
 367
 368         if (gen > 0) {
 369           // Now that we've removed a commit point, which
 370           // should have orphan'd at least one index file.
 371           // Open & close a writer and assert that it
 372           // actually removed something:
 373           int preCount = dir.listAll().length;
 374           writer = new IndexWriter(dir, newIndexWriterConfig(
 375               TEST_VERSION_CURRENT,
 376               new MockAnalyzer(random)).setOpenMode(
 377               OpenMode.APPEND).setIndexDeletionPolicy(policy));
 378           writer.close();
 379           int postCount = dir.listAll().length;
 380           assertTrue(postCount < preCount);
 381         }
 382       }
 383
 384       dir.close();
 385     }
 386   }
 387
 388   /* Uses KeepAllDeletionPolicy to keep all commits around,
 389    * then, opens a new IndexWriter on a previous commit
 390    * point. */
 391   public void testOpenPriorSnapshot() throws IOException {
 392     // Never deletes a commit
 393     KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();
 394
 395     Directory dir = newDirectory();
 396     policy.dir = dir;
 397
 398     IndexWriter writer = new IndexWriter(
 399         dir,
 400         newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
 401             setIndexDeletionPolicy(policy).
 402             setMaxBufferedDocs(2).
 403             setMergePolicy(newLogMergePolicy(10))
 404     );
 405     for(int i=0;i<10;i++) {
 406       addDoc(writer);
 407       if ((1+i)%2 == 0)
 408         writer.commit();
 409     }
 410     writer.close();
 411
 412     Collection<IndexCommit> commits = IndexReader.listCommits(dir);
 413     assertEquals(5, commits.size());
 414     IndexCommit lastCommit = null;
 415     for (final IndexCommit commit : commits) {
 416       if (lastCommit == null || commit.getGeneration() > lastCommit.getGeneration())
 417         lastCommit = commit;
 418     }
 419     assertTrue(lastCommit != null);
 420
 421     // Now add 1 doc and optimize
 422     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT,
 423         new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
 424     addDoc(writer);
 425     assertEquals(11, writer.numDocs());
 426     writer.optimize();
 427     writer.close();
 428
 429     assertEquals(6, IndexReader.listCommits(dir).size());
 430
 431     // Now open writer on the commit just before optimize:
 432     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
 433         .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
 434     assertEquals(10, writer.numDocs());
 435
 436     // Should undo our rollback:
 437     writer.rollback();
 438
 439     IndexReader r = IndexReader.open(dir, true);
 440     // Still optimized, still 11 docs
 441     assertTrue(r.isOptimized());
 442     assertEquals(11, r.numDocs());
 443     r.close();
 444
 445     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
 446         .setIndexDeletionPolicy(policy).setIndexCommit(lastCommit));
 447     assertEquals(10, writer.numDocs());
 448     // Commits the rollback:
 449     writer.close();
 450
 451     // Now 8 because we made another commit
 452     assertEquals(7, IndexReader.listCommits(dir).size());
 453
 454     r = IndexReader.open(dir, true);
 455     // Not optimized because we rolled it back, and now only
 456     // 10 docs
 457     assertTrue(!r.isOptimized());
 458     assertEquals(10, r.numDocs());
 459     r.close();
 460
 461     // Reoptimize
 462     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexDeletionPolicy(policy));
 463     writer.optimize();
 464     writer.close();
 465
 466     r = IndexReader.open(dir, true);
 467     assertTrue(r.isOptimized());
 468     assertEquals(10, r.numDocs());
 469     r.close();
 470
 471     // Now open writer on the commit just before optimize,
 472     // but this time keeping only the last commit:
 473     writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setIndexCommit(lastCommit));
 474     assertEquals(10, writer.numDocs());
 475
 476     // Reader still sees optimized index, because writer
 477     // opened on the prior commit has not yet committed:
 478     r = IndexReader.open(dir, true);
 479     assertTrue(r.isOptimized());
 480     assertEquals(10, r.numDocs());
 481     r.close();
 482
 483     writer.close();
 484
 485     // Now reader sees unoptimized index:
 486     r = IndexReader.open(dir, true);
 487     assertTrue(!r.isOptimized());
 488     assertEquals(10, r.numDocs());
 489     r.close();
 490
 491     dir.close();
 492   }
 493
 494
 495   /* Test keeping NO commit points.  This is a viable and
 496    * useful case eg where you want to build a big index and
 497    * you know there are no readers.
 498    */
 499   public void testKeepNoneOnInitDeletionPolicy() throws IOException {
 500     for(int pass=0;pass<2;pass++) {
 501
 502       boolean useCompoundFile = (pass % 2) != 0;
 503
 504       KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy();
 505
 506       Directory dir = newDirectory();
 507
 508       IndexWriterConfig conf = newIndexWriterConfig(
 509           TEST_VERSION_CURRENT, new MockAnalyzer(random))
 510           .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
 511           .setMaxBufferedDocs(10);
 512       MergePolicy mp = conf.getMergePolicy();
 513       if (mp instanceof LogMergePolicy) {
 514         setUseCompoundFile(mp, useCompoundFile);
 515       }
 516       IndexWriter writer = new IndexWriter(dir, conf);
 517       for(int i=0;i<107;i++) {
 518         addDoc(writer);
 519       }
 520       writer.close();
 521
 522       conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
 523           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
 524       mp = conf.getMergePolicy();
 525       if (mp instanceof LogMergePolicy) {
 526         setUseCompoundFile(mp, true);
 527       }
 528       writer = new IndexWriter(dir, conf);
 529       writer.optimize();
 530       writer.close();
 531
 532       assertEquals(1, policy.numOnInit);
 533       // If we are not auto committing then there should
 534       // be exactly 2 commits (one per close above):
 535       assertEquals(2, policy.numOnCommit);
 536
 537       // Simplistic check: just verify the index is in fact
 538       // readable:
 539       IndexReader reader = IndexReader.open(dir, true);
 540       reader.close();
 541
 542       dir.close();
 543     }
 544   }
 545
 546   /*
 547    * Test a deletion policy that keeps last N commits.
 548    */
 549   public void testKeepLastNDeletionPolicy() throws IOException {
 550     final int N = 5;
 551
 552     for(int pass=0;pass<2;pass++) {
 553
 554       boolean useCompoundFile = (pass % 2) != 0;
 555
 556       Directory dir = newDirectory();
 557
 558       KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
 559
 560       for(int j=0;j<N+1;j++) {
 561         IndexWriterConfig conf = newIndexWriterConfig(
 562             TEST_VERSION_CURRENT, new MockAnalyzer(random))
 563             .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
 564             .setMaxBufferedDocs(10);
 565         MergePolicy mp = conf.getMergePolicy();
 566         if (mp instanceof LogMergePolicy) {
 567           setUseCompoundFile(mp, useCompoundFile);
 568         }
 569         IndexWriter writer = new IndexWriter(dir, conf);
 570         for(int i=0;i<17;i++) {
 571           addDoc(writer);
 572         }
 573         writer.optimize();
 574         writer.close();
 575       }
 576
 577       assertTrue(policy.numDelete > 0);
 578       assertEquals(N, policy.numOnInit);
 579       assertEquals(N+1, policy.numOnCommit);
 580
 581       // Simplistic check: just verify only the past N segments_N's still
 582       // exist, and, I can open a reader on each:
 583       dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 584       long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 585       for(int i=0;i<N+1;i++) {
 586         try {
 587           IndexReader reader = IndexReader.open(dir, true);
 588           reader.close();
 589           if (i == N) {
 590             fail("should have failed on commits prior to last " + N);
 591           }
 592         } catch (IOException e) {
 593           if (i != N) {
 594             throw e;
 595           }
 596         }
 597         if (i < N) {
 598           dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 599         }
 600         gen--;
 601       }
 602
 603       dir.close();
 604     }
 605   }
 606
 607   /*
 608    * Test a deletion policy that keeps last N commits
 609    * around, with reader doing deletes.
 610    */
 611   public void testKeepLastNDeletionPolicyWithReader() throws IOException {
 612     final int N = 10;
 613
 614     for(int pass=0;pass<2;pass++) {
 615
 616       boolean useCompoundFile = (pass % 2) != 0;
 617
 618       KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
 619
 620       Directory dir = newDirectory();
 621       IndexWriterConfig conf = newIndexWriterConfig(
 622           TEST_VERSION_CURRENT, new MockAnalyzer(random))
 623         .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
 624       MergePolicy mp = conf.getMergePolicy();
 625       if (mp instanceof LogMergePolicy) {
 626         setUseCompoundFile(mp, useCompoundFile);
 627       }
 628       IndexWriter writer = new IndexWriter(dir, conf);
 629       writer.close();
 630       Term searchTerm = new Term("content", "aaa");
 631       Query query = new TermQuery(searchTerm);
 632
 633       for(int i=0;i<N+1;i++) {
 634         if (VERBOSE) {
 635           System.out.println("\nTEST: cycle i=" + i);
 636         }
 637         conf = newIndexWriterConfig(
 638             TEST_VERSION_CURRENT, new MockAnalyzer(random))
 639           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy).setMergePolicy(newLogMergePolicy());
 640         mp = conf.getMergePolicy();
 641         if (mp instanceof LogMergePolicy) {
 642           setUseCompoundFile(mp, useCompoundFile);
 643         }
 644         writer = new IndexWriter(dir, conf);
 645         writer.setInfoStream(VERBOSE ? System.out : null);
 646         for(int j=0;j<17;j++) {
 647           addDoc(writer);
 648         }
 649         // this is a commit
 650         if (VERBOSE) {
 651           System.out.println("TEST: close writer");
 652         }
 653         writer.close();
 654         IndexReader reader = IndexReader.open(dir, policy, false);
 655         reader.deleteDocument(3*i+1);
 656         reader.setNorm(4*i+1, "content", 2.0F);
 657         IndexSearcher searcher = newSearcher(reader);
 658         ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
 659         assertEquals(16*(1+i), hits.length);
 660         // this is a commit
 661         if (VERBOSE) {
 662           System.out.println("TEST: close reader numOnCommit=" + policy.numOnCommit);
 663         }
 664         reader.close();
 665         searcher.close();
 666       }
 667       conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
 668           .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
 669       mp = conf.getMergePolicy();
 670       if (mp instanceof LogMergePolicy) {
 671         setUseCompoundFile(mp, useCompoundFile);
 672       }
 673       IndexReader r = IndexReader.open(dir);
 674       final boolean wasOptimized = r.isOptimized();
 675       r.close();
 676       writer = new IndexWriter(dir, conf);
 677       writer.optimize();
 678       // this is a commit
 679       writer.close();
 680
 681       assertEquals(2*(N+1)+1, policy.numOnInit);
 682       assertEquals(2*(N+2) - (wasOptimized ? 1:0), policy.numOnCommit);
 683
 684       IndexSearcher searcher = new IndexSearcher(dir, false);
 685       ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
 686       assertEquals(176, hits.length);
 687
 688       // Simplistic check: just verify only the past N segments_N's still
 689       // exist, and, I can open a reader on each:
 690       long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 691
 692       dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 693       int expectedCount = 176;
 694       searcher.close();
 695       for(int i=0;i<N+1;i++) {
 696         try {
 697           IndexReader reader = IndexReader.open(dir, true);
 698
 699           // Work backwards in commits on what the expected
 700           // count should be.
 701           searcher = newSearcher(reader);
 702           hits = searcher.search(query, null, 1000).scoreDocs;
 703           if (i > 1) {
 704             if (i % 2 == 0) {
 705               expectedCount += 1;
 706             } else {
 707               expectedCount -= 17;
 708             }
 709           }
 710           assertEquals(expectedCount, hits.length);
 711           searcher.close();
 712           reader.close();
 713           if (i == N) {
 714             fail("should have failed on commits before last 5");
 715           }
 716         } catch (IOException e) {
 717           if (i != N) {
 718             throw e;
 719           }
 720         }
 721         if (i < N) {
 722           dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 723         }
 724         gen--;
 725       }
 726       dir.close();
 727     }
 728   }
 729
 730   /*
 731    * Test a deletion policy that keeps last N commits
 732    * around, through creates.
 733    */
 734   public void testKeepLastNDeletionPolicyWithCreates() throws IOException {
 735
 736     final int N = 10;
 737
 738     for(int pass=0;pass<2;pass++) {
 739
 740       boolean useCompoundFile = (pass % 2) != 0;
 741
 742       KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);
 743
 744       Directory dir = newDirectory();
 745       IndexWriterConfig conf = newIndexWriterConfig(
 746           TEST_VERSION_CURRENT, new MockAnalyzer(random))
 747           .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
 748           .setMaxBufferedDocs(10);
 749       MergePolicy mp = conf.getMergePolicy();
 750       if (mp instanceof LogMergePolicy) {
 751         setUseCompoundFile(mp, useCompoundFile);
 752       }
 753       IndexWriter writer = new IndexWriter(dir, conf);
 754       writer.close();
 755       Term searchTerm = new Term("content", "aaa");
 756       Query query = new TermQuery(searchTerm);
 757
 758       for(int i=0;i<N+1;i++) {
 759
 760         conf = newIndexWriterConfig(
 761             TEST_VERSION_CURRENT, new MockAnalyzer(random))
 762             .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy)
 763             .setMaxBufferedDocs(10);
 764         mp = conf.getMergePolicy();
 765         if (mp instanceof LogMergePolicy) {
 766           setUseCompoundFile(mp, useCompoundFile);
 767         }
 768         writer = new IndexWriter(dir, conf);
 769         for(int j=0;j<17;j++) {
 770           addDoc(writer);
 771         }
 772         // this is a commit
 773         writer.close();
 774         IndexReader reader = IndexReader.open(dir, policy, false);
 775         reader.deleteDocument(3);
 776         reader.setNorm(5, "content", 2.0F);
 777         IndexSearcher searcher = newSearcher(reader);
 778         ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
 779         assertEquals(16, hits.length);
 780         // this is a commit
 781         reader.close();
 782         searcher.close();
 783
 784         writer = new IndexWriter(dir, newIndexWriterConfig(
 785             TEST_VERSION_CURRENT, new MockAnalyzer(random))
 786             .setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy));
 787         // This will not commit: there are no changes
 788         // pending because we opened for "create":
 789         writer.close();
 790       }
 791
 792       assertEquals(3*(N+1), policy.numOnInit);
 793       assertEquals(3*(N+1)+1, policy.numOnCommit);
 794
 795       IndexSearcher searcher = new IndexSearcher(dir, false);
 796       ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
 797       assertEquals(0, hits.length);
 798
 799       // Simplistic check: just verify only the past N segments_N's still
 800       // exist, and, I can open a reader on each:
 801       long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
 802
 803       dir.deleteFile(IndexFileNames.SEGMENTS_GEN);
 804       int expectedCount = 0;
 805
 806       for(int i=0;i<N+1;i++) {
 807         try {
 808           IndexReader reader = IndexReader.open(dir, true);
 809
 810           // Work backwards in commits on what the expected
 811           // count should be.
 812           searcher = newSearcher(reader);
 813           hits = searcher.search(query, null, 1000).scoreDocs;
 814           assertEquals(expectedCount, hits.length);
 815           searcher.close();
 816           if (expectedCount == 0) {
 817             expectedCount = 16;
 818           } else if (expectedCount == 16) {
 819             expectedCount = 17;
 820           } else if (expectedCount == 17) {
 821             expectedCount = 0;
 822           }
 823           reader.close();
 824           if (i == N) {
 825             fail("should have failed on commits before last " + N);
 826           }
 827         } catch (IOException e) {
 828           if (i != N) {
 829             throw e;
 830           }
 831         }
 832         if (i < N) {
 833           dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
 834         }
 835         gen--;
 836       }
 837
 838       dir.close();
 839     }
 840   }
 841
 842   private void addDoc(IndexWriter writer) throws IOException
 843   {
 844     Document doc = new Document();
 845     doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
 846     writer.addDocument(doc);
 847   }
 848 }