1 package org.apache.lucene.queryParser.standard;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.text.DateFormat;
22 import java.text.NumberFormat;
23 import java.text.ParseException;
24 import java.text.SimpleDateFormat;
25 import java.util.Collections;
26 import java.util.Date;
27 import java.util.HashMap;
28 import java.util.Locale;
30 import java.util.Random;
31 import java.util.TimeZone;
33 import org.apache.lucene.analysis.Analyzer;
34 import org.apache.lucene.analysis.MockAnalyzer;
35 import org.apache.lucene.document.Document;
36 import org.apache.lucene.document.Field;
37 import org.apache.lucene.document.NumericField;
38 import org.apache.lucene.index.IndexReader;
39 import org.apache.lucene.index.RandomIndexWriter;
40 import org.apache.lucene.queryParser.core.QueryNodeException;
41 import org.apache.lucene.queryParser.core.parser.EscapeQuerySyntax;
42 import org.apache.lucene.queryParser.standard.config.NumberDateFormat;
43 import org.apache.lucene.queryParser.standard.config.NumericConfig;
44 import org.apache.lucene.queryParser.standard.parser.EscapeQuerySyntaxImpl;
45 import org.apache.lucene.search.IndexSearcher;
46 import org.apache.lucene.search.Query;
47 import org.apache.lucene.search.TopDocs;
48 import org.apache.lucene.store.Directory;
49 import org.apache.lucene.util.LuceneTestCase;
50 import org.apache.lucene.util._TestUtil;
51 import org.junit.AfterClass;
52 import org.junit.BeforeClass;
53 import org.junit.Test;
55 public class TestNumericQueryParser extends LuceneTestCase {
57 private static enum NumberType {
58 NEGATIVE, ZERO, POSITIVE;
61 final private static int[] DATE_STYLES = {DateFormat.FULL, DateFormat.LONG,
62 DateFormat.MEDIUM, DateFormat.SHORT};
64 final private static int PRECISION_STEP = 8;
65 final private static String FIELD_NAME = "field";
66 private static Locale LOCALE;
67 private static TimeZone TIMEZONE;
68 private static Map<String,Number> RANDOM_NUMBER_MAP;
69 final private static EscapeQuerySyntax ESCAPER = new EscapeQuerySyntaxImpl();
70 final private static String DATE_FIELD_NAME = "date";
71 private static int DATE_STYLE;
72 private static int TIME_STYLE;
74 private static Analyzer ANALYZER;
76 private static NumberFormat NUMBER_FORMAT;
78 private static StandardQueryParser qp;
80 private static NumberDateFormat DATE_FORMAT;
82 private static Directory directory = null;
83 private static IndexReader reader = null;
84 private static IndexSearcher searcher = null;
86 private static boolean checkDateFormatSanity(DateFormat dateFormat, long date)
87 throws ParseException {
88 return date == dateFormat.parse(dateFormat.format(new Date(date)))
93 public static void beforeClass() throws Exception {
94 ANALYZER = new MockAnalyzer(random);
96 qp = new StandardQueryParser(ANALYZER);
98 final HashMap<String,Number> randomNumberMap = new HashMap<String,Number>();
100 SimpleDateFormat dateFormat;
102 boolean dateFormatSanityCheckPass;
106 fail("This test has problems to find a sane random DateFormat/NumberFormat. Stopped trying after 100 iterations.");
109 dateFormatSanityCheckPass = true;
110 LOCALE = randomLocale(random);
111 TIMEZONE = randomTimeZone(random);
112 DATE_STYLE = randomDateStyle(random);
113 TIME_STYLE = randomDateStyle(random);
115 // assumes localized date pattern will have at least year, month, day,
117 dateFormat = (SimpleDateFormat) DateFormat.getDateTimeInstance(
118 DATE_STYLE, TIME_STYLE, LOCALE);
120 // not all date patterns includes era, full year, timezone and second,
121 // so we add them here
122 dateFormat.applyPattern(dateFormat.toPattern() + " G s Z yyyy");
123 dateFormat.setTimeZone(TIMEZONE);
125 DATE_FORMAT = new NumberDateFormat(dateFormat);
128 randomDate = random.nextLong();
130 // prune date value so it doesn't pass in insane values to some
132 randomDate = randomDate % 3400000000000l;
134 // truncate to second
135 randomDate = (randomDate / 1000L) * 1000L;
137 // only positive values
138 randomDate = Math.abs(randomDate);
139 } while (randomDate == 0L);
141 dateFormatSanityCheckPass &= checkDateFormatSanity(dateFormat, randomDate);
143 dateFormatSanityCheckPass &= checkDateFormatSanity(dateFormat, 0);
145 dateFormatSanityCheckPass &= checkDateFormatSanity(dateFormat,
149 } while (!dateFormatSanityCheckPass);
151 NUMBER_FORMAT = NumberFormat.getNumberInstance(LOCALE);
152 NUMBER_FORMAT.setMaximumFractionDigits((random.nextInt() & 20) + 1);
153 NUMBER_FORMAT.setMinimumFractionDigits((random.nextInt() & 20) + 1);
154 NUMBER_FORMAT.setMaximumIntegerDigits((random.nextInt() & 20) + 1);
155 NUMBER_FORMAT.setMinimumIntegerDigits((random.nextInt() & 20) + 1);
162 while ((randomLong = normalizeNumber(Math.abs(random.nextLong()))
165 while ((randomDouble = normalizeNumber(Math.abs(random.nextDouble()))
166 .doubleValue()) == 0.0)
168 while ((randomFloat = normalizeNumber(Math.abs(random.nextFloat()))
169 .floatValue()) == 0.0f)
171 while ((randomInt = normalizeNumber(Math.abs(random.nextInt())).intValue()) == 0)
174 randomNumberMap.put(NumericField.DataType.LONG.name(), randomLong);
175 randomNumberMap.put(NumericField.DataType.INT.name(), randomInt);
176 randomNumberMap.put(NumericField.DataType.FLOAT.name(), randomFloat);
177 randomNumberMap.put(NumericField.DataType.DOUBLE.name(), randomDouble);
178 randomNumberMap.put(DATE_FIELD_NAME, randomDate);
180 RANDOM_NUMBER_MAP = Collections.unmodifiableMap(randomNumberMap);
182 directory = newDirectory();
183 RandomIndexWriter writer = new RandomIndexWriter(random, directory,
184 newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
185 .setMaxBufferedDocs(_TestUtil.nextInt(random, 50, 1000))
186 .setMergePolicy(newLogMergePolicy()));
188 Document doc = new Document();
189 HashMap<String,NumericConfig> numericConfigMap = new HashMap<String,NumericConfig>();
190 HashMap<String,NumericField> numericFieldMap = new HashMap<String,NumericField>();
191 qp.setNumericConfigMap(numericConfigMap);
193 for (NumericField.DataType type : NumericField.DataType.values()) {
194 numericConfigMap.put(type.name(), new NumericConfig(PRECISION_STEP,
195 NUMBER_FORMAT, type));
197 NumericField field = new NumericField(type.name(), PRECISION_STEP,
198 Field.Store.YES, true);
200 numericFieldMap.put(type.name(), field);
205 numericConfigMap.put(DATE_FIELD_NAME, new NumericConfig(PRECISION_STEP,
206 DATE_FORMAT, NumericField.DataType.LONG));
207 NumericField dateField = new NumericField(DATE_FIELD_NAME, PRECISION_STEP,
208 Field.Store.YES, true);
209 numericFieldMap.put(DATE_FIELD_NAME, dateField);
212 for (NumberType numberType : NumberType.values()) {
213 setFieldValues(numberType, numericFieldMap);
214 if (VERBOSE) System.out.println("Indexing document: " + doc);
215 writer.addDocument(doc);
218 reader = writer.getReader();
219 searcher = newSearcher(reader);
224 private static Number getNumberType(NumberType numberType, String fieldName) {
226 if (numberType == null) {
230 switch (numberType) {
233 return RANDOM_NUMBER_MAP.get(fieldName);
236 Number number = RANDOM_NUMBER_MAP.get(fieldName);
238 if (NumericField.DataType.LONG.name().equals(fieldName)
239 || DATE_FIELD_NAME.equals(fieldName)) {
240 number = -number.longValue();
242 } else if (NumericField.DataType.DOUBLE.name().equals(fieldName)) {
243 number = -number.doubleValue();
245 } else if (NumericField.DataType.FLOAT.name().equals(fieldName)) {
246 number = -number.floatValue();
248 } else if (NumericField.DataType.INT.name().equals(fieldName)) {
249 number = -number.intValue();
252 throw new IllegalArgumentException("field name not found: "
265 private static void setFieldValues(NumberType numberType,
266 HashMap<String,NumericField> numericFieldMap) {
268 Number number = getNumberType(numberType, NumericField.DataType.DOUBLE
270 numericFieldMap.get(NumericField.DataType.DOUBLE.name()).setDoubleValue(
271 number.doubleValue());
273 number = getNumberType(numberType, NumericField.DataType.INT.name());
274 numericFieldMap.get(NumericField.DataType.INT.name()).setIntValue(
277 number = getNumberType(numberType, NumericField.DataType.LONG.name());
278 numericFieldMap.get(NumericField.DataType.LONG.name()).setLongValue(
281 number = getNumberType(numberType, NumericField.DataType.FLOAT.name());
282 numericFieldMap.get(NumericField.DataType.FLOAT.name()).setFloatValue(
283 number.floatValue());
285 number = getNumberType(numberType, DATE_FIELD_NAME);
286 numericFieldMap.get(DATE_FIELD_NAME).setLongValue(number.longValue());
290 private static int randomDateStyle(Random random) {
291 return DATE_STYLES[random.nextInt(DATE_STYLES.length)];
295 public void testInclusiveNumericRange() throws Exception {
296 assertRangeQuery(NumberType.ZERO, NumberType.ZERO, true, true, 1);
297 assertRangeQuery(NumberType.ZERO, NumberType.POSITIVE, true, true, 2);
298 assertRangeQuery(NumberType.NEGATIVE, NumberType.ZERO, true, true, 2);
299 assertRangeQuery(NumberType.NEGATIVE, NumberType.POSITIVE, true, true, 3);
300 assertRangeQuery(NumberType.NEGATIVE, NumberType.NEGATIVE, true, true, 1);
304 // // test disabled since standard syntax parser does not work with inclusive and
305 // // exclusive at the same time
306 // public void testInclusiveLowerNumericRange() throws Exception {
307 // assertRangeQuery(NumberType.NEGATIVE, NumberType.ZERO, false, true, 1);
308 // assertRangeQuery(NumberType.ZERO, NumberType.POSITIVE, false, true, 1);
309 // assertRangeQuery(NumberType.NEGATIVE, NumberType.POSITIVE, false, true, 2);
310 // assertRangeQuery(NumberType.NEGATIVE, NumberType.NEGATIVE, false, true, 0);
314 // // test disabled since standard syntax parser does not work with inclusive and
315 // // exclusive at the same time
316 // public void testInclusiveUpperNumericRange() throws Exception {
317 // assertRangeQuery(NumberType.NEGATIVE, NumberType.ZERO, true, false, 1);
318 // assertRangeQuery(NumberType.ZERO, NumberType.POSITIVE, true, false, 1);
319 // assertRangeQuery(NumberType.NEGATIVE, NumberType.POSITIVE, true, false, 2);
320 // assertRangeQuery(NumberType.NEGATIVE, NumberType.NEGATIVE, true, false, 0);
324 public void testExclusiveNumericRange() throws Exception {
325 assertRangeQuery(NumberType.ZERO, NumberType.ZERO, false, false, 0);
326 assertRangeQuery(NumberType.ZERO, NumberType.POSITIVE, false, false, 0);
327 assertRangeQuery(NumberType.NEGATIVE, NumberType.ZERO, false, false, 0);
328 assertRangeQuery(NumberType.NEGATIVE, NumberType.POSITIVE, false, false, 1);
329 assertRangeQuery(NumberType.NEGATIVE, NumberType.NEGATIVE, false, false, 0);
333 //// test disabled since standard syntax parser does not work with open range
334 // public void testOpenRangeNumericQuery() throws Exception {
335 // assertOpenRangeQuery(NumberType.ZERO, "<", 1);
336 // assertOpenRangeQuery(NumberType.POSITIVE, "<", 2);
337 // assertOpenRangeQuery(NumberType.NEGATIVE, "<", 0);
339 // assertOpenRangeQuery(NumberType.ZERO, "<=", 2);
340 // assertOpenRangeQuery(NumberType.POSITIVE, "<=", 3);
341 // assertOpenRangeQuery(NumberType.NEGATIVE, "<=", 1);
343 // assertOpenRangeQuery(NumberType.ZERO, ">", 1);
344 // assertOpenRangeQuery(NumberType.POSITIVE, ">", 0);
345 // assertOpenRangeQuery(NumberType.NEGATIVE, ">", 2);
347 // assertOpenRangeQuery(NumberType.ZERO, ">=", 2);
348 // assertOpenRangeQuery(NumberType.POSITIVE, ">=", 1);
349 // assertOpenRangeQuery(NumberType.NEGATIVE, ">=", 3);
351 // assertOpenRangeQuery(NumberType.NEGATIVE, "=", 1);
352 // assertOpenRangeQuery(NumberType.ZERO, "=", 1);
353 // assertOpenRangeQuery(NumberType.POSITIVE, "=", 1);
355 // assertRangeQuery(NumberType.NEGATIVE, null, true, true, 3);
356 // assertRangeQuery(NumberType.NEGATIVE, null, false, true, 2);
357 // assertRangeQuery(NumberType.POSITIVE, null, true, false, 1);
358 // assertRangeQuery(NumberType.ZERO, null, false, false, 1);
360 // assertRangeQuery(null, NumberType.POSITIVE, true, true, 3);
361 // assertRangeQuery(null, NumberType.POSITIVE, true, false, 2);
362 // assertRangeQuery(null, NumberType.NEGATIVE, false, true, 1);
363 // assertRangeQuery(null, NumberType.ZERO, false, false, 1);
365 // assertRangeQuery(null, null, false, false, 3);
366 // assertRangeQuery(null, null, true, true, 3);
371 public void testSimpleNumericQuery() throws Exception {
372 assertSimpleQuery(NumberType.ZERO, 1);
373 assertSimpleQuery(NumberType.POSITIVE, 1);
374 assertSimpleQuery(NumberType.NEGATIVE, 1);
377 public void assertRangeQuery(NumberType lowerType, NumberType upperType,
378 boolean lowerInclusive, boolean upperInclusive, int expectedDocCount)
379 throws QueryNodeException, IOException {
381 StringBuilder sb = new StringBuilder();
383 String lowerInclusiveStr = (lowerInclusive ? "[" : "{");
384 String upperInclusiveStr = (upperInclusive ? "]" : "}");
386 for (NumericField.DataType type : NumericField.DataType.values()) {
387 String lowerStr = numberToString(getNumberType(lowerType, type.name()));
388 String upperStr = numberToString(getNumberType(upperType, type.name()));
390 sb.append("+").append(type.name()).append(':').append(lowerInclusiveStr)
391 .append('"').append(lowerStr).append("\" TO \"").append(upperStr)
392 .append('"').append(upperInclusiveStr).append(' ');
395 Number lowerDateNumber = getNumberType(lowerType, DATE_FIELD_NAME);
396 Number upperDateNumber = getNumberType(upperType, DATE_FIELD_NAME);
400 if (lowerDateNumber != null) {
401 lowerDateStr = ESCAPER.escape(
402 DATE_FORMAT.format(new Date(lowerDateNumber.longValue())), LOCALE,
403 EscapeQuerySyntax.Type.STRING).toString();
409 if (upperDateNumber != null) {
410 upperDateStr = ESCAPER.escape(
411 DATE_FORMAT.format(new Date(upperDateNumber.longValue())), LOCALE,
412 EscapeQuerySyntax.Type.STRING).toString();
418 sb.append("+").append(DATE_FIELD_NAME).append(':')
419 .append(lowerInclusiveStr).append('"').append(lowerDateStr).append(
420 "\" TO \"").append(upperDateStr).append('"').append(
423 testQuery(sb.toString(), expectedDocCount);
427 public void assertOpenRangeQuery(NumberType boundType, String operator, int expectedDocCount)
428 throws QueryNodeException, IOException {
430 StringBuilder sb = new StringBuilder();
432 for (NumericField.DataType type : NumericField.DataType.values()) {
433 String boundStr = numberToString(getNumberType(boundType, type.name()));
435 sb.append("+").append(type.name()).append(operator).append('"').append(boundStr).append('"').append(' ');
438 String boundDateStr = ESCAPER.escape(
439 DATE_FORMAT.format(new Date(getNumberType(boundType, DATE_FIELD_NAME)
440 .longValue())), LOCALE, EscapeQuerySyntax.Type.STRING).toString();
442 sb.append("+").append(DATE_FIELD_NAME).append(operator).append('"').append(boundDateStr).append('"');
444 testQuery(sb.toString(), expectedDocCount);
447 public void assertSimpleQuery(NumberType numberType, int expectedDocCount)
448 throws QueryNodeException, IOException {
449 StringBuilder sb = new StringBuilder();
451 for (NumericField.DataType type : NumericField.DataType.values()) {
452 String numberStr = numberToString(getNumberType(numberType, type.name()));
453 sb.append('+').append(type.name()).append(":\"").append(numberStr)
457 String dateStr = ESCAPER.escape(
458 DATE_FORMAT.format(new Date(getNumberType(numberType, DATE_FIELD_NAME)
459 .longValue())), LOCALE, EscapeQuerySyntax.Type.STRING).toString();
461 sb.append('+').append(DATE_FIELD_NAME).append(":\"").append(dateStr)
464 testQuery(sb.toString(), expectedDocCount);
468 private void testQuery(String queryStr, int expectedDocCount)
469 throws QueryNodeException, IOException {
470 if (VERBOSE) System.out.println("Parsing: " + queryStr);
472 Query query = qp.parse(queryStr, FIELD_NAME);
473 if (VERBOSE) System.out.println("Querying: " + query);
474 TopDocs topDocs = searcher.search(query, 1000);
476 String msg = "Query <" + queryStr + "> retrieved " + topDocs.totalHits
477 + " document(s), " + expectedDocCount + " document(s) expected.";
479 if (VERBOSE) System.out.println(msg);
481 assertEquals(msg, expectedDocCount, topDocs.totalHits);
484 private static String numberToString(Number number) {
485 return number == null ? "*" : ESCAPER.escape(NUMBER_FORMAT.format(number),
486 LOCALE, EscapeQuerySyntax.Type.STRING).toString();
489 private static Number normalizeNumber(Number number) throws ParseException {
490 return NUMBER_FORMAT.parse(NUMBER_FORMAT.format(number));
494 public static void afterClass() throws Exception {