--- /dev/null
+package org.apache.lucene.benchmark.byTask.tasks;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.benchmark.byTask.PerfRunData;
+import org.apache.lucene.benchmark.byTask.utils.Config;
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.IndexCommit;
+import org.apache.lucene.index.IndexDeletionPolicy;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.NoDeletionPolicy;
+import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.NoMergeScheduler;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.index.TieredMergePolicy;
+import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.util.Version;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+
+/**
+ * Create an index. <br>
+ * Other side effects: index writer object in perfRunData is set. <br>
+ * Relevant properties: <code>merge.factor (default 10),
+ * max.buffered (default no flush), max.field.length (default
+ * 10,000 tokens), max.field.length, compound (default true), ram.flush.mb [default 0],
+ * merge.policy (default org.apache.lucene.index.LogByteSizeMergePolicy),
+ * merge.scheduler (default
+ * org.apache.lucene.index.ConcurrentMergeScheduler),
+ * concurrent.merge.scheduler.max.thread.count and
+ * concurrent.merge.scheduler.max.merge.count (defaults per
+ * ConcurrentMergeScheduler) </code>.
+ * <p>
+ * This task also supports a "writer.info.stream" property with the following
+ * values:
+ * <ul>
+ * <li>SystemOut - sets {@link IndexWriter#setInfoStream(java.io.PrintStream)}
+ * to {@link System#out}.
+ * <li>SystemErr - sets {@link IndexWriter#setInfoStream(java.io.PrintStream)}
+ * to {@link System#err}.
+ * <li><file_name> - attempts to create a file given that name and sets
+ * {@link IndexWriter#setInfoStream(java.io.PrintStream)} to that file. If this
+ * denotes an invalid file name, or some error occurs, an exception will be
+ * thrown.
+ * </ul>
+ */
+public class CreateIndexTask extends PerfTask {
+
+ public CreateIndexTask(PerfRunData runData) {
+ super(runData);
+ }
+
+ public static IndexDeletionPolicy getIndexDeletionPolicy(Config config) {
+ String deletionPolicyName = config.get("deletion.policy", "org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy");
+ if (deletionPolicyName.equals(NoDeletionPolicy.class.getName())) {
+ return NoDeletionPolicy.INSTANCE;
+ } else {
+ try {
+ return Class.forName(deletionPolicyName).asSubclass(IndexDeletionPolicy.class).newInstance();
+ } catch (Exception e) {
+ throw new RuntimeException("unable to instantiate class '" + deletionPolicyName + "' as IndexDeletionPolicy", e);
+ }
+ }
+ }
+
+ @Override
+ public int doLogic() throws IOException {
+ PerfRunData runData = getRunData();
+ Config config = runData.getConfig();
+ runData.setIndexWriter(configureWriter(config, runData, OpenMode.CREATE, null));
+ return 1;
+ }
+
+ public static IndexWriterConfig createWriterConfig(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) {
+ Version version = Version.valueOf(config.get("writer.version", Version.LUCENE_31.toString()));
+ IndexWriterConfig iwConf = new IndexWriterConfig(version, runData.getAnalyzer());
+ iwConf.setOpenMode(mode);
+ IndexDeletionPolicy indexDeletionPolicy = getIndexDeletionPolicy(config);
+ iwConf.setIndexDeletionPolicy(indexDeletionPolicy);
+ if(commit != null)
+ iwConf.setIndexCommit(commit);
+
+
+ final String mergeScheduler = config.get("merge.scheduler",
+ "org.apache.lucene.index.ConcurrentMergeScheduler");
+ if (mergeScheduler.equals(NoMergeScheduler.class.getName())) {
+ iwConf.setMergeScheduler(NoMergeScheduler.INSTANCE);
+ } else {
+ try {
+ iwConf.setMergeScheduler(Class.forName(mergeScheduler).asSubclass(MergeScheduler.class).newInstance());
+ } catch (Exception e) {
+ throw new RuntimeException("unable to instantiate class '" + mergeScheduler + "' as merge scheduler", e);
+ }
+
+ if (mergeScheduler.equals("org.apache.lucene.index.ConcurrentMergeScheduler")) {
+ ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) iwConf.getMergeScheduler();
+ int v = config.get("concurrent.merge.scheduler.max.thread.count", -1);
+ if (v != -1) {
+ cms.setMaxThreadCount(v);
+ }
+ v = config.get("concurrent.merge.scheduler.max.merge.count", -1);
+ if (v != -1) {
+ cms.setMaxMergeCount(v);
+ }
+ }
+ }
+
+ final String mergePolicy = config.get("merge.policy",
+ "org.apache.lucene.index.LogByteSizeMergePolicy");
+ boolean isCompound = config.get("compound", true);
+ if (mergePolicy.equals(NoMergePolicy.class.getName())) {
+ iwConf.setMergePolicy(isCompound ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES);
+ } else {
+ try {
+ iwConf.setMergePolicy(Class.forName(mergePolicy).asSubclass(MergePolicy.class).newInstance());
+ } catch (Exception e) {
+ throw new RuntimeException("unable to instantiate class '" + mergePolicy + "' as merge policy", e);
+ }
+ if(iwConf.getMergePolicy() instanceof LogMergePolicy) {
+ LogMergePolicy logMergePolicy = (LogMergePolicy) iwConf.getMergePolicy();
+ logMergePolicy.setUseCompoundFile(isCompound);
+ logMergePolicy.setMergeFactor(config.get("merge.factor",OpenIndexTask.DEFAULT_MERGE_PFACTOR));
+ } else if(iwConf.getMergePolicy() instanceof TieredMergePolicy) {
+ TieredMergePolicy tieredMergePolicy = (TieredMergePolicy) iwConf.getMergePolicy();
+ tieredMergePolicy.setUseCompoundFile(isCompound);
+ }
+ }
+ final double ramBuffer = config.get("ram.flush.mb",OpenIndexTask.DEFAULT_RAM_FLUSH_MB);
+ final int maxBuffered = config.get("max.buffered",OpenIndexTask.DEFAULT_MAX_BUFFERED);
+ if (maxBuffered == IndexWriterConfig.DISABLE_AUTO_FLUSH) {
+ iwConf.setRAMBufferSizeMB(ramBuffer);
+ iwConf.setMaxBufferedDocs(maxBuffered);
+ } else {
+ iwConf.setMaxBufferedDocs(maxBuffered);
+ iwConf.setRAMBufferSizeMB(ramBuffer);
+ }
+
+ return iwConf;
+ }
+
+ public static IndexWriter configureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) throws CorruptIndexException, LockObtainFailedException, IOException {
+ IndexWriter writer = new IndexWriter(runData.getDirectory(), createWriterConfig(config, runData, mode, commit));
+ writer.setMaxFieldLength(config.get("max.field.length", OpenIndexTask.DEFAULT_MAX_FIELD_LENGTH));
+
+ String infoStreamVal = config.get("writer.info.stream", null);
+ if (infoStreamVal != null) {
+ if (infoStreamVal.equals("SystemOut")) {
+ writer.setInfoStream(System.out);
+ } else if (infoStreamVal.equals("SystemErr")) {
+ writer.setInfoStream(System.err);
+ } else {
+ File f = new File(infoStreamVal).getAbsoluteFile();
+ writer.setInfoStream(new PrintStream(new BufferedOutputStream(new FileOutputStream(f))));
+ }
+ }
+ return writer;
+ }
+}
fnp / pylucene.git / blobdiff