1 package org.apache.lucene.util;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.BufferedOutputStream;
21 import java.io.ByteArrayOutputStream;
23 import java.io.FileOutputStream;
24 import java.io.IOException;
25 import java.io.InputStream;
26 import java.io.OutputStream;
27 import java.io.PrintStream;
28 import java.lang.reflect.Method;
29 import java.util.Enumeration;
30 import java.util.HashMap;
32 import java.util.Random;
33 import java.util.zip.ZipEntry;
34 import java.util.zip.ZipFile;
36 import org.junit.Assert;
38 import org.apache.lucene.document.Document;
39 import org.apache.lucene.document.Field;
40 import org.apache.lucene.document.Fieldable;
41 import org.apache.lucene.index.CheckIndex;
42 import org.apache.lucene.index.ConcurrentMergeScheduler;
43 import org.apache.lucene.index.IndexWriter;
44 import org.apache.lucene.index.LogMergePolicy;
45 import org.apache.lucene.index.MergePolicy;
46 import org.apache.lucene.index.MergeScheduler;
47 import org.apache.lucene.index.TieredMergePolicy;
48 import org.apache.lucene.search.FieldDoc;
49 import org.apache.lucene.search.ScoreDoc;
50 import org.apache.lucene.search.TopDocs;
51 import org.apache.lucene.store.Directory;
53 public class _TestUtil {
55 /** Returns temp dir, based on String arg in its name;
56 * does not create the directory. */
57 public static File getTempDir(String desc) {
59 File f = createTempFile(desc, "tmp", LuceneTestCase.TEMP_DIR);
61 LuceneTestCase.registerTempFile(f);
63 } catch (IOException e) {
64 throw new RuntimeException(e);
69 * Deletes a directory and everything underneath it.
71 public static void rmDir(File dir) throws IOException {
73 for (File f : dir.listFiles()) {
74 if (f.isDirectory()) {
78 throw new IOException("could not delete " + f);
83 throw new IOException("could not delete " + dir);
89 * Convenience method: Unzip zipName + ".zip" under destDir, removing destDir first
91 public static void unzip(File zipName, File destDir) throws IOException {
93 ZipFile zipFile = new ZipFile(zipName);
95 Enumeration<? extends ZipEntry> entries = zipFile.entries();
100 LuceneTestCase.registerTempFile(destDir);
102 while (entries.hasMoreElements()) {
103 ZipEntry entry = entries.nextElement();
105 InputStream in = zipFile.getInputStream(entry);
106 File targetFile = new File(destDir, entry.getName());
107 if (entry.isDirectory()) {
108 // allow unzipping with directory structure
111 if (targetFile.getParentFile()!=null) {
112 // be on the safe side: do not rely on that directories are always extracted
113 // before their children (although this makes sense, but is it guaranteed?)
114 targetFile.getParentFile().mkdirs();
116 OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile));
118 byte[] buffer = new byte[8192];
120 while((len = in.read(buffer)) >= 0) {
121 out.write(buffer, 0, len);
132 public static void syncConcurrentMerges(IndexWriter writer) {
133 syncConcurrentMerges(writer.getConfig().getMergeScheduler());
136 public static void syncConcurrentMerges(MergeScheduler ms) {
137 if (ms instanceof ConcurrentMergeScheduler)
138 ((ConcurrentMergeScheduler) ms).sync();
141 /** This runs the CheckIndex tool on the index in. If any
142 * issues are hit, a RuntimeException is thrown; else,
143 * true is returned. */
144 public static CheckIndex.Status checkIndex(Directory dir) throws IOException {
145 ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
147 CheckIndex checker = new CheckIndex(dir);
148 checker.setInfoStream(new PrintStream(bos));
149 CheckIndex.Status indexStatus = checker.checkIndex();
150 if (indexStatus == null || indexStatus.clean == false) {
151 System.out.println("CheckIndex failed");
152 System.out.println(bos.toString());
153 throw new RuntimeException("CheckIndex failed");
159 /** Use only for testing.
160 * @deprecated -- in 3.0 we can use Arrays.toString
163 public static String arrayToString(int[] array) {
164 StringBuilder buf = new StringBuilder();
166 for(int i=0;i<array.length;i++) {
170 buf.append(array[i]);
173 return buf.toString();
176 /** Use only for testing.
177 * @deprecated -- in 3.0 we can use Arrays.toString
180 public static String arrayToString(Object[] array) {
181 StringBuilder buf = new StringBuilder();
183 for(int i=0;i<array.length;i++) {
187 buf.append(array[i]);
190 return buf.toString();
193 public static String randomSimpleString(Random r) {
194 final int end = r.nextInt(10);
199 final char[] buffer = new char[end];
200 for (int i = 0; i < end; i++) {
201 buffer[i] = (char) _TestUtil.nextInt(r, 97, 102);
203 return new String(buffer, 0, end);
206 /** Returns random string, including full unicode range. */
207 public static String randomUnicodeString(Random r) {
208 return randomUnicodeString(r, 20);
212 * Returns a random string up to a certain length.
214 public static String randomUnicodeString(Random r, int maxLength) {
215 final int end = r.nextInt(maxLength);
220 final char[] buffer = new char[end];
221 randomFixedLengthUnicodeString(r, buffer, 0, buffer.length);
222 return new String(buffer, 0, end);
226 * Fills provided char[] with valid random unicode code
229 public static void randomFixedLengthUnicodeString(Random random, char[] chars, int offset, int length) {
231 final int end = offset + length;
233 final int t = random.nextInt(5);
234 if (0 == t && i < length - 1) {
235 // Make a surrogate pair
237 chars[i++] = (char) nextInt(random, 0xd800, 0xdbff);
239 chars[i++] = (char) nextInt(random, 0xdc00, 0xdfff);
241 chars[i++] = (char) random.nextInt(0x80);
243 chars[i++] = (char) nextInt(random, 0x80, 0x7ff);
245 chars[i++] = (char) nextInt(random, 0x800, 0xd7ff);
247 chars[i++] = (char) nextInt(random, 0xe000, 0xfffe);
252 private static final int[] blockStarts = {
253 0x0000, 0x0080, 0x0100, 0x0180, 0x0250, 0x02B0, 0x0300, 0x0370, 0x0400,
254 0x0500, 0x0530, 0x0590, 0x0600, 0x0700, 0x0750, 0x0780, 0x07C0, 0x0800,
255 0x0900, 0x0980, 0x0A00, 0x0A80, 0x0B00, 0x0B80, 0x0C00, 0x0C80, 0x0D00,
256 0x0D80, 0x0E00, 0x0E80, 0x0F00, 0x1000, 0x10A0, 0x1100, 0x1200, 0x1380,
257 0x13A0, 0x1400, 0x1680, 0x16A0, 0x1700, 0x1720, 0x1740, 0x1760, 0x1780,
258 0x1800, 0x18B0, 0x1900, 0x1950, 0x1980, 0x19E0, 0x1A00, 0x1A20, 0x1B00,
259 0x1B80, 0x1C00, 0x1C50, 0x1CD0, 0x1D00, 0x1D80, 0x1DC0, 0x1E00, 0x1F00,
260 0x2000, 0x2070, 0x20A0, 0x20D0, 0x2100, 0x2150, 0x2190, 0x2200, 0x2300,
261 0x2400, 0x2440, 0x2460, 0x2500, 0x2580, 0x25A0, 0x2600, 0x2700, 0x27C0,
262 0x27F0, 0x2800, 0x2900, 0x2980, 0x2A00, 0x2B00, 0x2C00, 0x2C60, 0x2C80,
263 0x2D00, 0x2D30, 0x2D80, 0x2DE0, 0x2E00, 0x2E80, 0x2F00, 0x2FF0, 0x3000,
264 0x3040, 0x30A0, 0x3100, 0x3130, 0x3190, 0x31A0, 0x31C0, 0x31F0, 0x3200,
265 0x3300, 0x3400, 0x4DC0, 0x4E00, 0xA000, 0xA490, 0xA4D0, 0xA500, 0xA640,
266 0xA6A0, 0xA700, 0xA720, 0xA800, 0xA830, 0xA840, 0xA880, 0xA8E0, 0xA900,
267 0xA930, 0xA960, 0xA980, 0xAA00, 0xAA60, 0xAA80, 0xABC0, 0xAC00, 0xD7B0,
268 0xE000, 0xF900, 0xFB00, 0xFB50, 0xFE00, 0xFE10,
269 0xFE20, 0xFE30, 0xFE50, 0xFE70, 0xFF00, 0xFFF0,
270 0x10000, 0x10080, 0x10100, 0x10140, 0x10190, 0x101D0, 0x10280, 0x102A0,
271 0x10300, 0x10330, 0x10380, 0x103A0, 0x10400, 0x10450, 0x10480, 0x10800,
272 0x10840, 0x10900, 0x10920, 0x10A00, 0x10A60, 0x10B00, 0x10B40, 0x10B60,
273 0x10C00, 0x10E60, 0x11080, 0x12000, 0x12400, 0x13000, 0x1D000, 0x1D100,
274 0x1D200, 0x1D300, 0x1D360, 0x1D400, 0x1F000, 0x1F030, 0x1F100, 0x1F200,
275 0x20000, 0x2A700, 0x2F800, 0xE0000, 0xE0100, 0xF0000, 0x100000
278 private static final int[] blockEnds = {
279 0x007F, 0x00FF, 0x017F, 0x024F, 0x02AF, 0x02FF, 0x036F, 0x03FF, 0x04FF,
280 0x052F, 0x058F, 0x05FF, 0x06FF, 0x074F, 0x077F, 0x07BF, 0x07FF, 0x083F,
281 0x097F, 0x09FF, 0x0A7F, 0x0AFF, 0x0B7F, 0x0BFF, 0x0C7F, 0x0CFF, 0x0D7F,
282 0x0DFF, 0x0E7F, 0x0EFF, 0x0FFF, 0x109F, 0x10FF, 0x11FF, 0x137F, 0x139F,
283 0x13FF, 0x167F, 0x169F, 0x16FF, 0x171F, 0x173F, 0x175F, 0x177F, 0x17FF,
284 0x18AF, 0x18FF, 0x194F, 0x197F, 0x19DF, 0x19FF, 0x1A1F, 0x1AAF, 0x1B7F,
285 0x1BBF, 0x1C4F, 0x1C7F, 0x1CFF, 0x1D7F, 0x1DBF, 0x1DFF, 0x1EFF, 0x1FFF,
286 0x206F, 0x209F, 0x20CF, 0x20FF, 0x214F, 0x218F, 0x21FF, 0x22FF, 0x23FF,
287 0x243F, 0x245F, 0x24FF, 0x257F, 0x259F, 0x25FF, 0x26FF, 0x27BF, 0x27EF,
288 0x27FF, 0x28FF, 0x297F, 0x29FF, 0x2AFF, 0x2BFF, 0x2C5F, 0x2C7F, 0x2CFF,
289 0x2D2F, 0x2D7F, 0x2DDF, 0x2DFF, 0x2E7F, 0x2EFF, 0x2FDF, 0x2FFF, 0x303F,
290 0x309F, 0x30FF, 0x312F, 0x318F, 0x319F, 0x31BF, 0x31EF, 0x31FF, 0x32FF,
291 0x33FF, 0x4DBF, 0x4DFF, 0x9FFF, 0xA48F, 0xA4CF, 0xA4FF, 0xA63F, 0xA69F,
292 0xA6FF, 0xA71F, 0xA7FF, 0xA82F, 0xA83F, 0xA87F, 0xA8DF, 0xA8FF, 0xA92F,
293 0xA95F, 0xA97F, 0xA9DF, 0xAA5F, 0xAA7F, 0xAADF, 0xABFF, 0xD7AF, 0xD7FF,
294 0xF8FF, 0xFAFF, 0xFB4F, 0xFDFF, 0xFE0F, 0xFE1F,
295 0xFE2F, 0xFE4F, 0xFE6F, 0xFEFF, 0xFFEF, 0xFFFE, /* avoid 0xFFFF on 3.x */
296 0x1007F, 0x100FF, 0x1013F, 0x1018F, 0x101CF, 0x101FF, 0x1029F, 0x102DF,
297 0x1032F, 0x1034F, 0x1039F, 0x103DF, 0x1044F, 0x1047F, 0x104AF, 0x1083F,
298 0x1085F, 0x1091F, 0x1093F, 0x10A5F, 0x10A7F, 0x10B3F, 0x10B5F, 0x10B7F,
299 0x10C4F, 0x10E7F, 0x110CF, 0x123FF, 0x1247F, 0x1342F, 0x1D0FF, 0x1D1FF,
300 0x1D24F, 0x1D35F, 0x1D37F, 0x1D7FF, 0x1F02F, 0x1F09F, 0x1F1FF, 0x1F2FF,
301 0x2A6DF, 0x2B73F, 0x2FA1F, 0xE007F, 0xE01EF, 0xFFFFF, 0x10FFFF
304 /** Returns random string of length between 0-20 codepoints, all codepoints within the same unicode block. */
305 public static String randomRealisticUnicodeString(Random r) {
306 return randomRealisticUnicodeString(r, 20);
309 /** Returns random string of length up to maxLength codepoints , all codepoints within the same unicode block. */
310 public static String randomRealisticUnicodeString(Random r, int maxLength) {
311 return randomRealisticUnicodeString(r, 0, 20);
314 /** Returns random string of length between min and max codepoints, all codepoints within the same unicode block. */
315 public static String randomRealisticUnicodeString(Random r, int minLength, int maxLength) {
316 final int end = minLength + r.nextInt(maxLength);
317 final int block = r.nextInt(blockStarts.length);
318 StringBuilder sb = new StringBuilder();
319 for (int i = 0; i < end; i++)
320 sb.appendCodePoint(nextInt(r, blockStarts[block], blockEnds[block]));
321 return sb.toString();
324 /** Returns random string, with a given UTF-8 byte length*/
325 public static String randomFixedByteLengthUnicodeString(Random r, int length) {
327 final char[] buffer = new char[length*3];
330 for (; i < buffer.length && bytes != 0; i++) {
334 } else if (bytes >= 3) {
336 } else if (bytes >= 2) {
342 buffer[i] = (char) r.nextInt(0x80);
345 buffer[i] = (char) nextInt(r, 0x80, 0x7ff);
348 buffer[i] = (char) nextInt(r, 0x800, 0xd7ff);
351 buffer[i] = (char) nextInt(r, 0xe000, 0xfffe);
354 // Make a surrogate pair
356 buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff);
358 buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff);
363 return new String(buffer, 0, i);
366 /** start and end are BOTH inclusive */
367 public static int nextInt(Random r, int start, int end) {
368 return start + r.nextInt(end-start+1);
371 public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException {
372 String[] files = dir.listAll();
373 if (files.length > 1 || (files.length == 1 && !files[0].equals("write.lock"))) {
380 /** just tries to configure things to keep the open file
382 public static void reduceOpenFiles(IndexWriter w) {
383 // keep number of open files lowish
384 MergePolicy mp = w.getConfig().getMergePolicy();
385 if (mp instanceof LogMergePolicy) {
386 LogMergePolicy lmp = (LogMergePolicy) mp;
387 lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
388 } else if (mp instanceof TieredMergePolicy) {
389 TieredMergePolicy tmp = (TieredMergePolicy) mp;
390 tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
391 tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
394 MergeScheduler ms = w.getConfig().getMergeScheduler();
395 if (ms instanceof ConcurrentMergeScheduler) {
396 ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2);
397 ((ConcurrentMergeScheduler) ms).setMaxMergeCount(3);
401 /** Checks some basic behaviour of an AttributeImpl
402 * @param reflectedValues contains a map with "AttributeClass#key" as values
404 public static <T> void assertAttributeReflection(final AttributeImpl att, Map<String,T> reflectedValues) {
405 final Map<String,Object> map = new HashMap<String,Object>();
406 att.reflectWith(new AttributeReflector() {
407 public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
408 map.put(attClass.getName() + '#' + key, value);
411 Assert.assertEquals("Reflection does not produce same map", reflectedValues, map);
414 public static void keepFullyDeletedSegments(IndexWriter w) {
416 // Carefully invoke what is a package-private (test
417 // only, internal) method on IndexWriter:
418 Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments");
419 m.setAccessible(true);
421 } catch (Exception e) {
422 // Should not happen?
423 throw new RuntimeException(e);
428 * insecure, fast version of File.createTempFile
429 * uses Random instead of SecureRandom.
431 public static File createTempFile(String prefix, String suffix, File directory)
433 // Force a prefix null check first
434 if (prefix.length() < 3) {
435 throw new IllegalArgumentException("prefix must be 3");
437 String newSuffix = suffix == null ? ".tmp" : suffix;
440 result = genTempFile(prefix, newSuffix, directory);
441 } while (!result.createNewFile());
445 /* Temp file counter */
446 private static int counter = 0;
448 /* identify for differnt VM processes */
449 private static int counterBase = 0;
451 private static class TempFileLocker {};
452 private static TempFileLocker tempFileLocker = new TempFileLocker();
454 private static File genTempFile(String prefix, String suffix, File directory) {
457 synchronized (tempFileLocker) {
459 int newInt = new Random().nextInt();
460 counter = ((newInt / 65535) & 0xFFFF) + 0x2710;
461 counterBase = counter;
463 identify = counter++;
466 StringBuilder newName = new StringBuilder();
467 newName.append(prefix);
468 newName.append(counterBase);
469 newName.append(identify);
470 newName.append(suffix);
471 return new File(directory, newName.toString());
474 public static void assertEquals(TopDocs expected, TopDocs actual) {
475 Assert.assertEquals("wrong total hits", expected.totalHits, actual.totalHits);
476 Assert.assertEquals("wrong maxScore", expected.getMaxScore(), actual.getMaxScore(), 0.0);
477 Assert.assertEquals("wrong hit count", expected.scoreDocs.length, actual.scoreDocs.length);
478 for(int hitIDX=0;hitIDX<expected.scoreDocs.length;hitIDX++) {
479 final ScoreDoc expectedSD = expected.scoreDocs[hitIDX];
480 final ScoreDoc actualSD = actual.scoreDocs[hitIDX];
481 Assert.assertEquals("wrong hit docID", expectedSD.doc, actualSD.doc);
482 Assert.assertEquals("wrong hit score", expectedSD.score, actualSD.score, 0.0);
483 if (expectedSD instanceof FieldDoc) {
484 Assert.assertTrue(actualSD instanceof FieldDoc);
485 Assert.assertEquals("wrong sort field values",
486 ((FieldDoc) expectedSD).fields,
487 ((FieldDoc) actualSD).fields);
489 Assert.assertFalse(actualSD instanceof FieldDoc);
494 // NOTE: this is likely buggy, and cannot clone fields
495 // with tokenStreamValues, etc. Use at your own risk!!
497 // TODO: is there a pre-existing way to do this!!!
498 public static Document cloneDocument(Document doc1) {
499 final Document doc2 = new Document();
500 for(Fieldable f : doc1.getFields()) {
501 Field field1 = (Field) f;
503 Field field2 = new Field(field1.name(),
504 field1.stringValue(),
505 field1.isStored() ? Field.Store.YES : Field.Store.NO,
506 field1.isIndexed() ? (field1.isTokenized() ? Field.Index.ANALYZED : Field.Index.NOT_ANALYZED) : Field.Index.NO);
507 field2.setOmitNorms(field1.getOmitNorms());
508 field2.setIndexOptions(field1.getIndexOptions());