--- /dev/null
+package org.apache.lucene.search.vectorhighlight;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.util.AttributeImpl;
+
+public class IndexTimeSynonymTest extends AbstractTestCase {
+
+ public void testFieldTermStackIndex1wSearch1term() throws Exception {
+ makeIndex1w();
+
+ FieldQuery fq = new FieldQuery( tq( "Mac" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 1, stack.termList.size() );
+ assertEquals( "Mac(11,20,3)", stack.pop().toString() );
+ }
+
+ public void testFieldTermStackIndex1wSearch2terms() throws Exception {
+ makeIndex1w();
+
+ BooleanQuery bq = new BooleanQuery();
+ bq.add( tq( "Mac" ), Occur.SHOULD );
+ bq.add( tq( "MacBook" ), Occur.SHOULD );
+ FieldQuery fq = new FieldQuery( bq, true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 2, stack.termList.size() );
+ Set<String> expectedSet = new HashSet<String>();
+ expectedSet.add( "Mac(11,20,3)" );
+ expectedSet.add( "MacBook(11,20,3)" );
+ assertTrue( expectedSet.contains( stack.pop().toString() ) );
+ assertTrue( expectedSet.contains( stack.pop().toString() ) );
+ }
+
+ public void testFieldTermStackIndex1w2wSearch1term() throws Exception {
+ makeIndex1w2w();
+
+ FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 1, stack.termList.size() );
+ assertEquals( "pc(3,5,1)", stack.pop().toString() );
+ }
+
+ public void testFieldTermStackIndex1w2wSearch1phrase() throws Exception {
+ makeIndex1w2w();
+
+ FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 2, stack.termList.size() );
+ assertEquals( "personal(3,5,1)", stack.pop().toString() );
+ assertEquals( "computer(3,5,2)", stack.pop().toString() );
+ }
+
+ public void testFieldTermStackIndex1w2wSearch1partial() throws Exception {
+ makeIndex1w2w();
+
+ FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 1, stack.termList.size() );
+ assertEquals( "computer(3,5,2)", stack.pop().toString() );
+ }
+
+ public void testFieldTermStackIndex1w2wSearch1term1phrase() throws Exception {
+ makeIndex1w2w();
+
+ BooleanQuery bq = new BooleanQuery();
+ bq.add( tq( "pc" ), Occur.SHOULD );
+ bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
+ FieldQuery fq = new FieldQuery( bq, true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 3, stack.termList.size() );
+ Set<String> expectedSet = new HashSet<String>();
+ expectedSet.add( "pc(3,5,1)" );
+ expectedSet.add( "personal(3,5,1)" );
+ assertTrue( expectedSet.contains( stack.pop().toString() ) );
+ assertTrue( expectedSet.contains( stack.pop().toString() ) );
+ assertEquals( "computer(3,5,2)", stack.pop().toString() );
+ }
+
+ public void testFieldTermStackIndex2w1wSearch1term() throws Exception {
+ makeIndex2w1w();
+
+ FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 1, stack.termList.size() );
+ assertEquals( "pc(3,20,1)", stack.pop().toString() );
+ }
+
+ public void testFieldTermStackIndex2w1wSearch1phrase() throws Exception {
+ makeIndex2w1w();
+
+ FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 2, stack.termList.size() );
+ assertEquals( "personal(3,20,1)", stack.pop().toString() );
+ assertEquals( "computer(3,20,2)", stack.pop().toString() );
+ }
+
+ public void testFieldTermStackIndex2w1wSearch1partial() throws Exception {
+ makeIndex2w1w();
+
+ FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 1, stack.termList.size() );
+ assertEquals( "computer(3,20,2)", stack.pop().toString() );
+ }
+
+ public void testFieldTermStackIndex2w1wSearch1term1phrase() throws Exception {
+ makeIndex2w1w();
+
+ BooleanQuery bq = new BooleanQuery();
+ bq.add( tq( "pc" ), Occur.SHOULD );
+ bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
+ FieldQuery fq = new FieldQuery( bq, true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ assertEquals( 3, stack.termList.size() );
+ Set<String> expectedSet = new HashSet<String>();
+ expectedSet.add( "pc(3,20,1)" );
+ expectedSet.add( "personal(3,20,1)" );
+ assertTrue( expectedSet.contains( stack.pop().toString() ) );
+ assertTrue( expectedSet.contains( stack.pop().toString() ) );
+ assertEquals( "computer(3,20,2)", stack.pop().toString() );
+ }
+
+ public void testFieldPhraseListIndex1w2wSearch1phrase() throws Exception {
+ makeIndex1w2w();
+
+ FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ assertEquals( 1, fpl.phraseList.size() );
+ assertEquals( "personalcomputer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() );
+ assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
+ assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );
+ }
+
+ public void testFieldPhraseListIndex1w2wSearch1partial() throws Exception {
+ makeIndex1w2w();
+
+ FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ assertEquals( 1, fpl.phraseList.size() );
+ assertEquals( "computer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() );
+ assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
+ assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );
+ }
+
+ public void testFieldPhraseListIndex1w2wSearch1term1phrase() throws Exception {
+ makeIndex1w2w();
+
+ BooleanQuery bq = new BooleanQuery();
+ bq.add( tq( "pc" ), Occur.SHOULD );
+ bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
+ FieldQuery fq = new FieldQuery( bq, true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ assertEquals( 1, fpl.phraseList.size() );
+ assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,5))" ) > 0 );
+ assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
+ assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() );
+ }
+
+ public void testFieldPhraseListIndex2w1wSearch1term() throws Exception {
+ makeIndex2w1w();
+
+ FieldQuery fq = new FieldQuery( tq( "pc" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ assertEquals( 1, fpl.phraseList.size() );
+ assertEquals( "pc(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );
+ assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
+ assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
+ }
+
+ public void testFieldPhraseListIndex2w1wSearch1phrase() throws Exception {
+ makeIndex2w1w();
+
+ FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ assertEquals( 1, fpl.phraseList.size() );
+ assertEquals( "personalcomputer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );
+ assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
+ assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
+ }
+
+ public void testFieldPhraseListIndex2w1wSearch1partial() throws Exception {
+ makeIndex2w1w();
+
+ FieldQuery fq = new FieldQuery( tq( "computer" ), true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ assertEquals( 1, fpl.phraseList.size() );
+ assertEquals( "computer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() );
+ assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
+ assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
+ }
+
+ public void testFieldPhraseListIndex2w1wSearch1term1phrase() throws Exception {
+ makeIndex2w1w();
+
+ BooleanQuery bq = new BooleanQuery();
+ bq.add( tq( "pc" ), Occur.SHOULD );
+ bq.add( pqF( "personal", "computer" ), Occur.SHOULD );
+ FieldQuery fq = new FieldQuery( bq, true, true );
+ FieldTermStack stack = new FieldTermStack( reader, 0, F, fq );
+ FieldPhraseList fpl = new FieldPhraseList( stack, fq );
+ assertEquals( 1, fpl.phraseList.size() );
+ assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,20))" ) > 0 );
+ assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() );
+ assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() );
+ }
+
+ private void makeIndex1w() throws Exception {
+ // 11111111112
+ // 012345678901234567890
+ // I'll buy a Macintosh
+ // Mac
+ // MacBook
+ // 0 1 2 3
+ makeSynonymIndex( "I'll buy a Macintosh",
+ t("I'll",0,4),
+ t("buy",5,8),
+ t("a",9,10),
+ t("Macintosh",11,20),t("Mac",11,20,0),t("MacBook",11,20,0));
+ }
+
+ private void makeIndex1w2w() throws Exception {
+ // 1111111
+ // 01234567890123456
+ // My pc was broken
+ // personal computer
+ // 0 1 2 3
+ makeSynonymIndex( "My pc was broken",
+ t("My",0,2),
+ t("pc",3,5),t("personal",3,5,0),t("computer",3,5),
+ t("was",6,9),
+ t("broken",10,16));
+ }
+
+ private void makeIndex2w1w() throws Exception {
+ // 1111111111222222222233
+ // 01234567890123456789012345678901
+ // My personal computer was broken
+ // pc
+ // 0 1 2 3 4
+ makeSynonymIndex( "My personal computer was broken",
+ t("My",0,2),
+ t("personal",3,20),t("pc",3,20,0),t("computer",3,20),
+ t("was",21,24),
+ t("broken",25,31));
+ }
+
+ void makeSynonymIndex( String value, Token... tokens ) throws Exception {
+ Analyzer analyzer = new TokenArrayAnalyzer( tokens );
+ make1dmfIndex( analyzer, value );
+ }
+
+ public static Token t( String text, int startOffset, int endOffset ){
+ return t( text, startOffset, endOffset, 1 );
+ }
+
+ public static Token t( String text, int startOffset, int endOffset, int positionIncrement ){
+ Token token = new Token( text, startOffset, endOffset );
+ token.setPositionIncrement( positionIncrement );
+ return token;
+ }
+
+ public static final class TokenArrayAnalyzer extends Analyzer {
+ Token[] tokens;
+ public TokenArrayAnalyzer( Token... tokens ){
+ this.tokens = tokens;
+ }
+
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) {
+ final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class);
+ int p = 0;
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if( p >= tokens.length ) return false;
+ clearAttributes();
+ tokens[p++].copyTo(reusableToken);
+ return true;
+ }
+ };
+ return ts;
+ }
+ }
+}
fnp / pylucene.git / blobdiff