+ * This test case tests {@link PrecedenceQueryParser}. + *
+ *+ * It contains all tests from {@link QueryParserTestBase} with some adjusted to + * fit the precedence requirement, plus some precedence test cases. + *
+ * + * @see QueryParserTestBase + */ +// TODO: refactor this to actually extend that class, overriding the tests +// that it adjusts to fit the precedence requirement, adding its extra tests. +public class TestPrecedenceQueryParser extends LuceneTestCase { + + public static Analyzer qpAnalyzer = new QPTestAnalyzer(); + + public static final class QPTestFilter extends TokenFilter { + /** + * Filter which discards the token 'stop' and which expands the token + * 'phrase' into 'phrase1 phrase2' + */ + public QPTestFilter(TokenStream in) { + super(in); + } + + private boolean inPhrase = false; + + private int savedStart = 0; + private int savedEnd = 0; + + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + + @Override + public boolean incrementToken() throws IOException { + if (inPhrase) { + inPhrase = false; + termAtt.setEmpty().append("phrase2"); + offsetAtt.setOffset(savedStart, savedEnd); + return true; + } else + while (input.incrementToken()) + if (termAtt.toString().equals("phrase")) { + inPhrase = true; + savedStart = offsetAtt.startOffset(); + savedEnd = offsetAtt.endOffset(); + termAtt.setEmpty().append("phrase1"); + offsetAtt.setOffset(savedStart, savedEnd); + return true; + } else if (!termAtt.toString().equals("stop")) + return true; + return false; + } + + @Override + public void reset() throws IOException { + super.reset(); + this.inPhrase = false; + this.savedStart = 0; + this.savedEnd = 0; + } + } + + public static final class QPTestAnalyzer extends Analyzer { + + /** Filters LowerCaseTokenizer with StopFilter. */ + @Override + public final TokenStream tokenStream(String fieldName, Reader reader) { + return new QPTestFilter(new LowerCaseTokenizer(TEST_VERSION_CURRENT, reader)); + } + } + + private int originalMaxClauses; + + @Override + public void setUp() throws Exception { + super.setUp(); + originalMaxClauses = BooleanQuery.getMaxClauseCount(); + } + + public PrecedenceQueryParser getParser(Analyzer a) throws Exception { + if (a == null) + a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true); + PrecedenceQueryParser qp = new PrecedenceQueryParser(); + qp.setAnalyzer(a); + qp.setDefaultOperator(StandardQueryConfigHandler.Operator.OR); + return qp; + } + + public Query getQuery(String query, Analyzer a) throws Exception { + return getParser(a).parse(query, "field"); + } + + public void assertQueryEquals(String query, Analyzer a, String result) + throws Exception { + Query q = getQuery(query, a); + String s = q.toString("field"); + if (!s.equals(result)) { + fail("Query /" + query + "/ yielded /" + s + "/, expecting /" + result + + "/"); + } + } + + public void assertWildcardQueryEquals(String query, boolean lowercase, + String result) throws Exception { + PrecedenceQueryParser qp = getParser(null); + qp.setLowercaseExpandedTerms(lowercase); + Query q = qp.parse(query, "field"); + String s = q.toString("field"); + if (!s.equals(result)) { + fail("WildcardQuery /" + query + "/ yielded /" + s + "/, expecting /" + + result + "/"); + } + } + + public void assertWildcardQueryEquals(String query, String result) + throws Exception { + PrecedenceQueryParser qp = getParser(null); + Query q = qp.parse(query, "field"); + String s = q.toString("field"); + if (!s.equals(result)) { + fail("WildcardQuery /" + query + "/ yielded /" + s + "/, expecting /" + + result + "/"); + } + } + + public Query getQueryDOA(String query, Analyzer a) throws Exception { + if (a == null) + a = new MockAnalyzer(random, MockTokenizer.SIMPLE, true); + PrecedenceQueryParser qp = new PrecedenceQueryParser(); + qp.setAnalyzer(a); + qp.setDefaultOperator(StandardQueryConfigHandler.Operator.AND); + return qp.parse(query, "field"); + } + + public void assertQueryEqualsDOA(String query, Analyzer a, String result) + throws Exception { + Query q = getQueryDOA(query, a); + String s = q.toString("field"); + if (!s.equals(result)) { + fail("Query /" + query + "/ yielded /" + s + "/, expecting /" + result + + "/"); + } + } + + public void testSimple() throws Exception { + assertQueryEquals("term term term", null, "term term term"); + assertQueryEquals("türm term term", null, "türm term term"); + assertQueryEquals("ümlaut", null, "ümlaut"); + + assertQueryEquals("a AND b", null, "+a +b"); + assertQueryEquals("(a AND b)", null, "+a +b"); + assertQueryEquals("c OR (a AND b)", null, "c (+a +b)"); + assertQueryEquals("a AND NOT b", null, "+a -b"); + assertQueryEquals("a AND -b", null, "+a -b"); + assertQueryEquals("a AND !b", null, "+a -b"); + assertQueryEquals("a && b", null, "+a +b"); + assertQueryEquals("a && ! b", null, "+a -b"); + + assertQueryEquals("a OR b", null, "a b"); + assertQueryEquals("a || b", null, "a b"); + + assertQueryEquals("+term -term term", null, "+term -term term"); + assertQueryEquals("foo:term AND field:anotherTerm", null, + "+foo:term +anotherterm"); + assertQueryEquals("term AND \"phrase phrase\"", null, + "+term +\"phrase phrase\""); + assertQueryEquals("\"hello there\"", null, "\"hello there\""); + assertTrue(getQuery("a AND b", null) instanceof BooleanQuery); + assertTrue(getQuery("hello", null) instanceof TermQuery); + assertTrue(getQuery("\"hello there\"", null) instanceof PhraseQuery); + + assertQueryEquals("germ term^2.0", null, "germ term^2.0"); + assertQueryEquals("(term)^2.0", null, "term^2.0"); + assertQueryEquals("(germ term)^2.0", null, "(germ term)^2.0"); + assertQueryEquals("term^2.0", null, "term^2.0"); + assertQueryEquals("term^2", null, "term^2.0"); + assertQueryEquals("\"germ term\"^2.0", null, "\"germ term\"^2.0"); + assertQueryEquals("\"term germ\"^2", null, "\"term germ\"^2.0"); + + assertQueryEquals("(foo OR bar) AND (baz OR boo)", null, + "+(foo bar) +(baz boo)"); + assertQueryEquals("((a OR b) AND NOT c) OR d", null, "(+(a b) -c) d"); + assertQueryEquals("+(apple \"steve jobs\") -(foo bar baz)", null, + "+(apple \"steve jobs\") -(foo bar baz)"); + assertQueryEquals("+title:(dog OR cat) -author:\"bob dole\"", null, + "+(title:dog title:cat) -author:\"bob dole\""); + + PrecedenceQueryParser qp = new PrecedenceQueryParser(); + qp.setAnalyzer(new MockAnalyzer(random)); + // make sure OR is the default: + assertEquals(StandardQueryConfigHandler.Operator.OR, qp.getDefaultOperator()); + qp.setDefaultOperator(StandardQueryConfigHandler.Operator.AND); + assertEquals(StandardQueryConfigHandler.Operator.AND, qp.getDefaultOperator()); + qp.setDefaultOperator(StandardQueryConfigHandler.Operator.OR); + assertEquals(StandardQueryConfigHandler.Operator.OR, qp.getDefaultOperator()); + + assertQueryEquals("a OR !b", null, "a -b"); + assertQueryEquals("a OR ! b", null, "a -b"); + assertQueryEquals("a OR -b", null, "a -b"); + } + + public void testPunct() throws Exception { + Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false); + assertQueryEquals("a&b", a, "a&b"); + assertQueryEquals("a&&b", a, "a&&b"); + assertQueryEquals(".NET", a, ".NET"); + } + + public void testSlop() throws Exception { + assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2"); + assertQueryEquals("\"term germ\"~2 flork", null, "\"term germ\"~2 flork"); + assertQueryEquals("\"term\"~2", null, "term"); + assertQueryEquals("\" \"~2 germ", null, "germ"); + assertQueryEquals("\"term germ\"~2^2", null, "\"term germ\"~2^2.0"); + } + + public void testNumber() throws Exception { + // The numbers go away because SimpleAnalzyer ignores them + assertQueryEquals("3", null, ""); + assertQueryEquals("term 1.0 1 2", null, "term"); + assertQueryEquals("term term1 term2", null, "term term term"); + + Analyzer a = new MockAnalyzer(random); + assertQueryEquals("3", a, "3"); + assertQueryEquals("term 1.0 1 2", a, "term 1.0 1 2"); + assertQueryEquals("term term1 term2", a, "term term1 term2"); + } + + public void testWildcard() throws Exception { + assertQueryEquals("term*", null, "term*"); + assertQueryEquals("term*^2", null, "term*^2.0"); + assertQueryEquals("term~", null, "term~0.5"); + assertQueryEquals("term~0.7", null, "term~0.7"); + assertQueryEquals("term~^3", null, "term~0.5^3.0"); + assertQueryEquals("term^3~", null, "term~0.5^3.0"); + assertQueryEquals("term*germ", null, "term*germ"); + assertQueryEquals("term*germ^3", null, "term*germ^3.0"); + + assertTrue(getQuery("term*", null) instanceof PrefixQuery); + assertTrue(getQuery("term*^2", null) instanceof PrefixQuery); + assertTrue(getQuery("term~", null) instanceof FuzzyQuery); + assertTrue(getQuery("term~0.7", null) instanceof FuzzyQuery); + FuzzyQuery fq = (FuzzyQuery) getQuery("term~0.7", null); + assertEquals(0.7f, fq.getMinSimilarity(), 0.1f); + assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength()); + fq = (FuzzyQuery) getQuery("term~", null); + assertEquals(0.5f, fq.getMinSimilarity(), 0.1f); + assertEquals(FuzzyQuery.defaultPrefixLength, fq.getPrefixLength()); + try { + getQuery("term~1.1", null); // value > 1, throws exception + fail(); + } catch (ParseException pe) { + // expected exception + } + assertTrue(getQuery("term*germ", null) instanceof WildcardQuery); + + /* + * Tests to see that wild card terms are (or are not) properly lower-cased + * with propery parser configuration + */ + // First prefix queries: + // by default, convert to lowercase: + assertWildcardQueryEquals("Term*", true, "term*"); + // explicitly set lowercase: + assertWildcardQueryEquals("term*", true, "term*"); + assertWildcardQueryEquals("Term*", true, "term*"); + assertWildcardQueryEquals("TERM*", true, "term*"); + // explicitly disable lowercase conversion: + assertWildcardQueryEquals("term*", false, "term*"); + assertWildcardQueryEquals("Term*", false, "Term*"); + assertWildcardQueryEquals("TERM*", false, "TERM*"); + // Then 'full' wildcard queries: + // by default, convert to lowercase: + assertWildcardQueryEquals("Te?m", "te?m"); + // explicitly set lowercase: + assertWildcardQueryEquals("te?m", true, "te?m"); + assertWildcardQueryEquals("Te?m", true, "te?m"); + assertWildcardQueryEquals("TE?M", true, "te?m"); + assertWildcardQueryEquals("Te?m*gerM", true, "te?m*germ"); + // explicitly disable lowercase conversion: + assertWildcardQueryEquals("te?m", false, "te?m"); + assertWildcardQueryEquals("Te?m", false, "Te?m"); + assertWildcardQueryEquals("TE?M", false, "TE?M"); + assertWildcardQueryEquals("Te?m*gerM", false, "Te?m*gerM"); + // Fuzzy queries: + assertWildcardQueryEquals("Term~", "term~0.5"); + assertWildcardQueryEquals("Term~", true, "term~0.5"); + assertWildcardQueryEquals("Term~", false, "Term~0.5"); + // Range queries: + assertWildcardQueryEquals("[A TO C]", "[a TO c]"); + assertWildcardQueryEquals("[A TO C]", true, "[a TO c]"); + assertWildcardQueryEquals("[A TO C]", false, "[A TO C]"); + } + + public void testQPA() throws Exception { + assertQueryEquals("term term term", qpAnalyzer, "term term term"); + assertQueryEquals("term +stop term", qpAnalyzer, "term term"); + assertQueryEquals("term -stop term", qpAnalyzer, "term term"); + assertQueryEquals("drop AND stop AND roll", qpAnalyzer, "+drop +roll"); + assertQueryEquals("term phrase term", qpAnalyzer, + "term (phrase1 phrase2) term"); + // note the parens in this next assertion differ from the original + // QueryParser behavior + assertQueryEquals("term AND NOT phrase term", qpAnalyzer, + "(+term -(phrase1 phrase2)) term"); + assertQueryEquals("stop", qpAnalyzer, ""); + assertQueryEquals("stop OR stop AND stop", qpAnalyzer, ""); + assertTrue(getQuery("term term term", qpAnalyzer) instanceof BooleanQuery); + assertTrue(getQuery("term +stop", qpAnalyzer) instanceof TermQuery); + } + + public void testRange() throws Exception { + assertQueryEquals("[ a TO z]", null, "[a TO z]"); + assertTrue(getQuery("[ a TO z]", null) instanceof TermRangeQuery); + assertQueryEquals("[ a TO z ]", null, "[a TO z]"); + assertQueryEquals("{ a TO z}", null, "{a TO z}"); + assertQueryEquals("{ a TO z }", null, "{a TO z}"); + assertQueryEquals("{ a TO z }^2.0", null, "{a TO z}^2.0"); + assertQueryEquals("[ a TO z] OR bar", null, "[a TO z] bar"); + assertQueryEquals("[ a TO z] AND bar", null, "+[a TO z] +bar"); + assertQueryEquals("( bar blar { a TO z}) ", null, "bar blar {a TO z}"); + assertQueryEquals("gack ( bar blar { a TO z}) ", null, + "gack (bar blar {a TO z})"); + } + + private String escapeDateString(String s) { + if (s.contains(" ")) { + return "\"" + s + "\""; + } else { + return s; + } + } + + public String getDate(String s) throws Exception { + DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT); + return DateTools.dateToString(df.parse(s), DateTools.Resolution.DAY); + } + + private String getLocalizedDate(int year, int month, int day, + boolean extendLastDate) { + DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT); + Calendar calendar = new GregorianCalendar(); + calendar.set(year, month, day); + if (extendLastDate) { + calendar.set(Calendar.HOUR_OF_DAY, 23); + calendar.set(Calendar.MINUTE, 59); + calendar.set(Calendar.SECOND, 59); + calendar.set(Calendar.MILLISECOND, 999); + } + return df.format(calendar.getTime()); + } + + public void testDateRange() throws Exception { + String startDate = getLocalizedDate(2002, 1, 1, false); + String endDate = getLocalizedDate(2002, 1, 4, false); + Calendar endDateExpected = new GregorianCalendar(); + endDateExpected.set(2002, 1, 4, 23, 59, 59); + endDateExpected.set(Calendar.MILLISECOND, 999); + final String defaultField = "default"; + final String monthField = "month"; + final String hourField = "hour"; + PrecedenceQueryParser qp = new PrecedenceQueryParser(new MockAnalyzer(random)); + + // Don't set any date resolution and verify if DateField is used + assertDateRangeQueryEquals(qp, defaultField, startDate, endDate, + endDateExpected.getTime(), null); + + Map