1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
22 import org.apache.lucene.analysis.MockAnalyzer;
23 import org.apache.lucene.document.Document;
24 import org.apache.lucene.document.Field;
25 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
26 import org.apache.lucene.store.Directory;
28 import org.apache.lucene.util.LuceneTestCase;
30 public class TestIndexWriterMergePolicy extends LuceneTestCase {
32 // Test the normal case
33 public void testNormalCase() throws IOException {
34 Directory dir = newDirectory();
36 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
37 TEST_VERSION_CURRENT, new MockAnalyzer(random))
38 .setMaxBufferedDocs(10).setMergePolicy(new LogDocMergePolicy()));
40 for (int i = 0; i < 100; i++) {
42 checkInvariants(writer);
49 // Test to see if there is over merge
50 public void testNoOverMerge() throws IOException {
51 Directory dir = newDirectory();
53 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
54 TEST_VERSION_CURRENT, new MockAnalyzer(random))
55 .setMaxBufferedDocs(10).setMergePolicy(new LogDocMergePolicy()));
57 boolean noOverMerge = false;
58 for (int i = 0; i < 100; i++) {
60 checkInvariants(writer);
61 if (writer.getNumBufferedDocuments() + writer.getSegmentCount() >= 18) {
65 assertTrue(noOverMerge);
71 // Test the case where flush is forced after every addDoc
72 public void testForceFlush() throws IOException {
73 Directory dir = newDirectory();
75 LogDocMergePolicy mp = new LogDocMergePolicy();
76 mp.setMinMergeDocs(100);
77 mp.setMergeFactor(10);
78 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
79 TEST_VERSION_CURRENT, new MockAnalyzer(random))
80 .setMaxBufferedDocs(10).setMergePolicy(mp));
82 for (int i = 0; i < 100; i++) {
86 mp = new LogDocMergePolicy();
87 mp.setMergeFactor(10);
88 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
89 new MockAnalyzer(random)).setOpenMode(
90 OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(mp));
91 mp.setMinMergeDocs(100);
92 checkInvariants(writer);
99 // Test the case where mergeFactor changes
100 public void testMergeFactorChange() throws IOException {
101 Directory dir = newDirectory();
103 IndexWriter writer = new IndexWriter(
105 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
106 setMaxBufferedDocs(10).
107 setMergePolicy(newLogMergePolicy()).
108 setMergeScheduler(new SerialMergeScheduler())
111 writer.setInfoStream(VERBOSE ? System.out : null);
113 for (int i = 0; i < 250; i++) {
115 checkInvariants(writer);
118 ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(5);
120 // merge policy only fixes segments on levels where merges
121 // have been triggered, so check invariants after all adds
122 for (int i = 0; i < 10; i++) {
125 checkInvariants(writer);
131 // Test the case where both mergeFactor and maxBufferedDocs change
132 public void testMaxBufferedDocsChange() throws IOException {
133 Directory dir = newDirectory();
135 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
136 TEST_VERSION_CURRENT, new MockAnalyzer(random))
137 .setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy())
138 .setMergeScheduler(new SerialMergeScheduler()));
140 // leftmost* segment has 1 doc
141 // rightmost* segment has 100 docs
142 for (int i = 1; i <= 100; i++) {
143 for (int j = 0; j < i; j++) {
145 checkInvariants(writer);
149 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
150 new MockAnalyzer(random)).setOpenMode(
151 OpenMode.APPEND).setMaxBufferedDocs(101).setMergePolicy(new LogDocMergePolicy())
152 .setMergeScheduler(new SerialMergeScheduler()));
156 LogDocMergePolicy ldmp = new LogDocMergePolicy();
157 ldmp.setMergeFactor(10);
158 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
159 new MockAnalyzer(random)).setOpenMode(
160 OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new SerialMergeScheduler()));
162 // merge policy only fixes segments on levels where merges
163 // have been triggered, so check invariants after all adds
164 for (int i = 0; i < 100; i++) {
167 checkInvariants(writer);
169 for (int i = 100; i < 1000; i++) {
173 writer.waitForMerges();
175 checkInvariants(writer);
181 // Test the case where a merge results in no doc at all
182 public void testMergeDocCount0() throws IOException {
183 Directory dir = newDirectory();
185 LogDocMergePolicy ldmp = new LogDocMergePolicy();
186 ldmp.setMergeFactor(100);
187 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
188 TEST_VERSION_CURRENT, new MockAnalyzer(random))
189 .setMaxBufferedDocs(10).setMergePolicy(ldmp));
191 for (int i = 0; i < 250; i++) {
193 checkInvariants(writer);
197 IndexReader reader = IndexReader.open(dir, false);
198 reader.deleteDocuments(new Term("content", "aaa"));
201 ldmp = new LogDocMergePolicy();
202 ldmp.setMergeFactor(5);
203 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
204 new MockAnalyzer(random)).setOpenMode(
205 OpenMode.APPEND).setMaxBufferedDocs(10).setMergePolicy(ldmp).setMergeScheduler(new ConcurrentMergeScheduler()));
207 // merge factor is changed, so check invariants after all adds
208 for (int i = 0; i < 10; i++) {
212 writer.waitForMerges();
214 checkInvariants(writer);
215 assertEquals(10, writer.maxDoc());
221 private void addDoc(IndexWriter writer) throws IOException {
222 Document doc = new Document();
223 doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
224 writer.addDocument(doc);
227 private void checkInvariants(IndexWriter writer) throws IOException {
228 writer.waitForMerges();
229 int maxBufferedDocs = writer.getConfig().getMaxBufferedDocs();
230 int mergeFactor = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMergeFactor();
231 int maxMergeDocs = ((LogMergePolicy) writer.getConfig().getMergePolicy()).getMaxMergeDocs();
233 int ramSegmentCount = writer.getNumBufferedDocuments();
234 assertTrue(ramSegmentCount < maxBufferedDocs);
237 int upperBound = maxBufferedDocs;
240 int segmentCount = writer.getSegmentCount();
241 for (int i = segmentCount - 1; i >= 0; i--) {
242 int docCount = writer.getDocCount(i);
243 assertTrue("docCount=" + docCount + " lowerBound=" + lowerBound + " upperBound=" + upperBound + " i=" + i + " segmentCount=" + segmentCount + " index=" + writer.segString() + " config=" + writer.getConfig(), docCount > lowerBound);
245 if (docCount <= upperBound) {
248 if (upperBound * mergeFactor <= maxMergeDocs) {
249 assertTrue("maxMergeDocs=" + maxMergeDocs + "; numSegments=" + numSegments + "; upperBound=" + upperBound + "; mergeFactor=" + mergeFactor + "; segs=" + writer.segString() + " config=" + writer.getConfig(), numSegments < mergeFactor);
253 lowerBound = upperBound;
254 upperBound *= mergeFactor;
255 } while (docCount > upperBound);
259 if (upperBound * mergeFactor <= maxMergeDocs) {
260 assertTrue(numSegments < mergeFactor);