X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.4.0/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java diff --git a/lucene-java-3.4.0/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java b/lucene-java-3.4.0/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java deleted file mode 100644 index f31a5bd..0000000 --- a/lucene-java-3.4.0/lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java +++ /dev/null @@ -1,318 +0,0 @@ -package org.apache.lucene.search.vectorhighlight; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.io.Reader; -import java.util.HashSet; -import java.util.Set; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.util.AttributeImpl; - -public class IndexTimeSynonymTest extends AbstractTestCase { - - public void testFieldTermStackIndex1wSearch1term() throws Exception { - makeIndex1w(); - - FieldQuery fq = new FieldQuery( tq( "Mac" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 1, stack.termList.size() ); - assertEquals( "Mac(11,20,3)", stack.pop().toString() ); - } - - public void testFieldTermStackIndex1wSearch2terms() throws Exception { - makeIndex1w(); - - BooleanQuery bq = new BooleanQuery(); - bq.add( tq( "Mac" ), Occur.SHOULD ); - bq.add( tq( "MacBook" ), Occur.SHOULD ); - FieldQuery fq = new FieldQuery( bq, true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 2, stack.termList.size() ); - Set expectedSet = new HashSet(); - expectedSet.add( "Mac(11,20,3)" ); - expectedSet.add( "MacBook(11,20,3)" ); - assertTrue( expectedSet.contains( stack.pop().toString() ) ); - assertTrue( expectedSet.contains( stack.pop().toString() ) ); - } - - public void testFieldTermStackIndex1w2wSearch1term() throws Exception { - makeIndex1w2w(); - - FieldQuery fq = new FieldQuery( tq( "pc" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 1, stack.termList.size() ); - assertEquals( "pc(3,5,1)", stack.pop().toString() ); - } - - public void testFieldTermStackIndex1w2wSearch1phrase() throws Exception { - makeIndex1w2w(); - - FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 2, stack.termList.size() ); - assertEquals( "personal(3,5,1)", stack.pop().toString() ); - assertEquals( "computer(3,5,2)", stack.pop().toString() ); - } - - public void testFieldTermStackIndex1w2wSearch1partial() throws Exception { - makeIndex1w2w(); - - FieldQuery fq = new FieldQuery( tq( "computer" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 1, stack.termList.size() ); - assertEquals( "computer(3,5,2)", stack.pop().toString() ); - } - - public void testFieldTermStackIndex1w2wSearch1term1phrase() throws Exception { - makeIndex1w2w(); - - BooleanQuery bq = new BooleanQuery(); - bq.add( tq( "pc" ), Occur.SHOULD ); - bq.add( pqF( "personal", "computer" ), Occur.SHOULD ); - FieldQuery fq = new FieldQuery( bq, true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 3, stack.termList.size() ); - Set expectedSet = new HashSet(); - expectedSet.add( "pc(3,5,1)" ); - expectedSet.add( "personal(3,5,1)" ); - assertTrue( expectedSet.contains( stack.pop().toString() ) ); - assertTrue( expectedSet.contains( stack.pop().toString() ) ); - assertEquals( "computer(3,5,2)", stack.pop().toString() ); - } - - public void testFieldTermStackIndex2w1wSearch1term() throws Exception { - makeIndex2w1w(); - - FieldQuery fq = new FieldQuery( tq( "pc" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 1, stack.termList.size() ); - assertEquals( "pc(3,20,1)", stack.pop().toString() ); - } - - public void testFieldTermStackIndex2w1wSearch1phrase() throws Exception { - makeIndex2w1w(); - - FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 2, stack.termList.size() ); - assertEquals( "personal(3,20,1)", stack.pop().toString() ); - assertEquals( "computer(3,20,2)", stack.pop().toString() ); - } - - public void testFieldTermStackIndex2w1wSearch1partial() throws Exception { - makeIndex2w1w(); - - FieldQuery fq = new FieldQuery( tq( "computer" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 1, stack.termList.size() ); - assertEquals( "computer(3,20,2)", stack.pop().toString() ); - } - - public void testFieldTermStackIndex2w1wSearch1term1phrase() throws Exception { - makeIndex2w1w(); - - BooleanQuery bq = new BooleanQuery(); - bq.add( tq( "pc" ), Occur.SHOULD ); - bq.add( pqF( "personal", "computer" ), Occur.SHOULD ); - FieldQuery fq = new FieldQuery( bq, true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - assertEquals( 3, stack.termList.size() ); - Set expectedSet = new HashSet(); - expectedSet.add( "pc(3,20,1)" ); - expectedSet.add( "personal(3,20,1)" ); - assertTrue( expectedSet.contains( stack.pop().toString() ) ); - assertTrue( expectedSet.contains( stack.pop().toString() ) ); - assertEquals( "computer(3,20,2)", stack.pop().toString() ); - } - - public void testFieldPhraseListIndex1w2wSearch1phrase() throws Exception { - makeIndex1w2w(); - - FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - FieldPhraseList fpl = new FieldPhraseList( stack, fq ); - assertEquals( 1, fpl.phraseList.size() ); - assertEquals( "personalcomputer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() ); - assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); - assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() ); - } - - public void testFieldPhraseListIndex1w2wSearch1partial() throws Exception { - makeIndex1w2w(); - - FieldQuery fq = new FieldQuery( tq( "computer" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - FieldPhraseList fpl = new FieldPhraseList( stack, fq ); - assertEquals( 1, fpl.phraseList.size() ); - assertEquals( "computer(1.0)((3,5))", fpl.phraseList.get( 0 ).toString() ); - assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); - assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() ); - } - - public void testFieldPhraseListIndex1w2wSearch1term1phrase() throws Exception { - makeIndex1w2w(); - - BooleanQuery bq = new BooleanQuery(); - bq.add( tq( "pc" ), Occur.SHOULD ); - bq.add( pqF( "personal", "computer" ), Occur.SHOULD ); - FieldQuery fq = new FieldQuery( bq, true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - FieldPhraseList fpl = new FieldPhraseList( stack, fq ); - assertEquals( 1, fpl.phraseList.size() ); - assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,5))" ) > 0 ); - assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); - assertEquals( 5, fpl.phraseList.get( 0 ).getEndOffset() ); - } - - public void testFieldPhraseListIndex2w1wSearch1term() throws Exception { - makeIndex2w1w(); - - FieldQuery fq = new FieldQuery( tq( "pc" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - FieldPhraseList fpl = new FieldPhraseList( stack, fq ); - assertEquals( 1, fpl.phraseList.size() ); - assertEquals( "pc(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() ); - assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); - assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() ); - } - - public void testFieldPhraseListIndex2w1wSearch1phrase() throws Exception { - makeIndex2w1w(); - - FieldQuery fq = new FieldQuery( pqF( "personal", "computer" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - FieldPhraseList fpl = new FieldPhraseList( stack, fq ); - assertEquals( 1, fpl.phraseList.size() ); - assertEquals( "personalcomputer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() ); - assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); - assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() ); - } - - public void testFieldPhraseListIndex2w1wSearch1partial() throws Exception { - makeIndex2w1w(); - - FieldQuery fq = new FieldQuery( tq( "computer" ), true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - FieldPhraseList fpl = new FieldPhraseList( stack, fq ); - assertEquals( 1, fpl.phraseList.size() ); - assertEquals( "computer(1.0)((3,20))", fpl.phraseList.get( 0 ).toString() ); - assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); - assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() ); - } - - public void testFieldPhraseListIndex2w1wSearch1term1phrase() throws Exception { - makeIndex2w1w(); - - BooleanQuery bq = new BooleanQuery(); - bq.add( tq( "pc" ), Occur.SHOULD ); - bq.add( pqF( "personal", "computer" ), Occur.SHOULD ); - FieldQuery fq = new FieldQuery( bq, true, true ); - FieldTermStack stack = new FieldTermStack( reader, 0, F, fq ); - FieldPhraseList fpl = new FieldPhraseList( stack, fq ); - assertEquals( 1, fpl.phraseList.size() ); - assertTrue( fpl.phraseList.get( 0 ).toString().indexOf( "(1.0)((3,20))" ) > 0 ); - assertEquals( 3, fpl.phraseList.get( 0 ).getStartOffset() ); - assertEquals( 20, fpl.phraseList.get( 0 ).getEndOffset() ); - } - - private void makeIndex1w() throws Exception { - // 11111111112 - // 012345678901234567890 - // I'll buy a Macintosh - // Mac - // MacBook - // 0 1 2 3 - makeSynonymIndex( "I'll buy a Macintosh", - t("I'll",0,4), - t("buy",5,8), - t("a",9,10), - t("Macintosh",11,20),t("Mac",11,20,0),t("MacBook",11,20,0)); - } - - private void makeIndex1w2w() throws Exception { - // 1111111 - // 01234567890123456 - // My pc was broken - // personal computer - // 0 1 2 3 - makeSynonymIndex( "My pc was broken", - t("My",0,2), - t("pc",3,5),t("personal",3,5,0),t("computer",3,5), - t("was",6,9), - t("broken",10,16)); - } - - private void makeIndex2w1w() throws Exception { - // 1111111111222222222233 - // 01234567890123456789012345678901 - // My personal computer was broken - // pc - // 0 1 2 3 4 - makeSynonymIndex( "My personal computer was broken", - t("My",0,2), - t("personal",3,20),t("pc",3,20,0),t("computer",3,20), - t("was",21,24), - t("broken",25,31)); - } - - void makeSynonymIndex( String value, Token... tokens ) throws Exception { - Analyzer analyzer = new TokenArrayAnalyzer( tokens ); - make1dmfIndex( analyzer, value ); - } - - public static Token t( String text, int startOffset, int endOffset ){ - return t( text, startOffset, endOffset, 1 ); - } - - public static Token t( String text, int startOffset, int endOffset, int positionIncrement ){ - Token token = new Token( text, startOffset, endOffset ); - token.setPositionIncrement( positionIncrement ); - return token; - } - - public static final class TokenArrayAnalyzer extends Analyzer { - Token[] tokens; - public TokenArrayAnalyzer( Token... tokens ){ - this.tokens = tokens; - } - - @Override - public TokenStream tokenStream(String fieldName, Reader reader) { - TokenStream ts = new TokenStream(Token.TOKEN_ATTRIBUTE_FACTORY) { - final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class); - int p = 0; - - @Override - public boolean incrementToken() throws IOException { - if( p >= tokens.length ) return false; - clearAttributes(); - tokens[p++].copyTo(reusableToken); - return true; - } - }; - return ts; - } - } -}