1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
22 import org.apache.lucene.analysis.MockAnalyzer;
23 import org.apache.lucene.document.Document;
24 import org.apache.lucene.document.Field;
25 import org.apache.lucene.document.Field.Index;
26 import org.apache.lucene.document.Field.Store;
27 import org.apache.lucene.document.Field.TermVector;
28 import org.apache.lucene.index.IndexWriterConfig.OpenMode;
29 import org.apache.lucene.store.Directory;
30 import org.apache.lucene.store.MockDirectoryWrapper;
31 import org.apache.lucene.util.LuceneTestCase;
32 import org.apache.lucene.util._TestUtil;
34 public class TestIndexWriterOptimize extends LuceneTestCase {
35 public void testOptimizeMaxNumSegments() throws IOException {
37 MockDirectoryWrapper dir = newDirectory();
39 final Document doc = new Document();
40 doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
41 final int incrMin = TEST_NIGHTLY ? 15 : 40;
42 for(int numDocs=10;numDocs<500;numDocs += _TestUtil.nextInt(random, incrMin, 5*incrMin)) {
43 LogDocMergePolicy ldmp = new LogDocMergePolicy();
44 ldmp.setMinMergeDocs(1);
45 ldmp.setMergeFactor(5);
46 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
47 TEST_VERSION_CURRENT, new MockAnalyzer(random))
48 .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(2).setMergePolicy(
50 for(int j=0;j<numDocs;j++)
51 writer.addDocument(doc);
54 SegmentInfos sis = new SegmentInfos();
56 final int segCount = sis.size();
58 ldmp = new LogDocMergePolicy();
59 ldmp.setMergeFactor(5);
60 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
61 new MockAnalyzer(random)).setMergePolicy(ldmp));
65 sis = new SegmentInfos();
67 final int optSegCount = sis.size();
70 assertEquals(segCount, optSegCount);
72 assertEquals(3, optSegCount);
77 public void testOptimizeMaxNumSegments2() throws IOException {
78 MockDirectoryWrapper dir = newDirectory();
80 final Document doc = new Document();
81 doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
83 LogDocMergePolicy ldmp = new LogDocMergePolicy();
84 ldmp.setMinMergeDocs(1);
85 ldmp.setMergeFactor(4);
86 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
87 TEST_VERSION_CURRENT, new MockAnalyzer(random))
88 .setMaxBufferedDocs(2).setMergePolicy(ldmp).setMergeScheduler(new ConcurrentMergeScheduler()));
90 for(int iter=0;iter<10;iter++) {
92 writer.addDocument(doc);
95 writer.waitForMerges();
98 SegmentInfos sis = new SegmentInfos();
101 final int segCount = sis.size();
105 writer.waitForMerges();
107 sis = new SegmentInfos();
109 final int optSegCount = sis.size();
112 assertEquals(segCount, optSegCount);
114 assertEquals(7, optSegCount);
121 * Make sure optimize doesn't use any more than 1X
122 * starting index size as its temporary free space
125 public void testOptimizeTempSpaceUsage() throws IOException {
127 MockDirectoryWrapper dir = newDirectory();
128 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(10).setMergePolicy(newLogMergePolicy()));
130 System.out.println("TEST: config1=" + writer.getConfig());
133 for(int j=0;j<500;j++) {
134 TestIndexWriter.addDocWithIndex(writer, j);
136 final int termIndexInterval = writer.getConfig().getTermIndexInterval();
137 // force one extra segment w/ different doc store so
138 // we see the doc stores get merged
140 TestIndexWriter.addDocWithIndex(writer, 500);
144 System.out.println("TEST: start disk usage");
146 long startDiskUsage = 0;
147 String[] files = dir.listAll();
148 for(int i=0;i<files.length;i++) {
149 startDiskUsage += dir.fileLength(files[i]);
151 System.out.println(files[i] + ": " + dir.fileLength(files[i]));
155 dir.resetMaxUsedSizeInBytes();
156 dir.setTrackDiskUsage(true);
158 // Import to use same term index interval else a
159 // smaller one here could increase the disk usage and
160 // cause a false failure:
161 writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND).setTermIndexInterval(termIndexInterval).setMergePolicy(newLogMergePolicy()));
162 writer.setInfoStream(VERBOSE ? System.out : null);
165 long maxDiskUsage = dir.getMaxUsedSizeInBytes();
166 assertTrue("optimize used too much temporary space: starting usage was " + startDiskUsage + " bytes; max temp usage was " + maxDiskUsage + " but should have been " + (4*startDiskUsage) + " (= 4X starting usage)",
167 maxDiskUsage <= 4*startDiskUsage);
171 // Test calling optimize(false) whereby optimize is kicked
172 // off but we don't wait for it to finish (but
173 // writer.close()) does wait
174 public void testBackgroundOptimize() throws IOException {
176 Directory dir = newDirectory();
177 for(int pass=0;pass<2;pass++) {
178 IndexWriter writer = new IndexWriter(
180 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
181 setOpenMode(OpenMode.CREATE).
182 setMaxBufferedDocs(2).
183 setMergePolicy(newLogMergePolicy(51))
185 Document doc = new Document();
186 doc.add(newField("field", "aaa", Store.NO, Index.NOT_ANALYZED));
187 for(int i=0;i<100;i++)
188 writer.addDocument(doc);
189 writer.optimize(false);
193 IndexReader reader = IndexReader.open(dir, true);
194 assertTrue(reader.isOptimized());
197 // Get another segment to flush so we can verify it is
198 // NOT included in the optimization
199 writer.addDocument(doc);
200 writer.addDocument(doc);
203 IndexReader reader = IndexReader.open(dir, true);
204 assertTrue(!reader.isOptimized());
207 SegmentInfos infos = new SegmentInfos();
209 assertEquals(2, infos.size());