--- /dev/null
+package org.apache.lucene.search.vectorhighlight;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Collection;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenizer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Index;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.document.Field.TermVector;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.IndexWriterConfig.OpenMode;
+import org.apache.lucene.queryParser.QueryParser;
+import org.apache.lucene.search.DisjunctionMaxQuery;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+public abstract class AbstractTestCase extends LuceneTestCase {
+
+ protected final String F = "f";
+ protected final String F1 = "f1";
+ protected final String F2 = "f2";
+ protected Directory dir;
+ protected Analyzer analyzerW;
+ protected Analyzer analyzerB;
+ protected Analyzer analyzerK;
+ protected IndexReader reader;
+ protected QueryParser paW;
+ protected QueryParser paB;
+
+ protected static final String[] shortMVValues = {
+ "",
+ "",
+ "a b c",
+ "", // empty data in multi valued field
+ "d e"
+ };
+
+ protected static final String[] longMVValues = {
+ "Followings are the examples of customizable parameters and actual examples of customization:",
+ "The most search engines use only one of these methods. Even the search engines that says they can use the both methods basically"
+ };
+
+ // test data for LUCENE-1448 bug
+ protected static final String[] biMVValues = {
+ "\nLucene/Solr does not require such additional hardware.",
+ "\nWhen you talk about processing speed, the"
+ };
+
+ protected static final String[] strMVValues = {
+ "abc",
+ "defg",
+ "hijkl"
+ };
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ analyzerW = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
+ analyzerB = new BigramAnalyzer();
+ analyzerK = new MockAnalyzer(random, MockTokenizer.KEYWORD, false);
+ paW = new QueryParser(TEST_VERSION_CURRENT, F, analyzerW );
+ paB = new QueryParser(TEST_VERSION_CURRENT, F, analyzerB );
+ dir = newDirectory();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ if( reader != null ){
+ reader.close();
+ reader = null;
+ }
+ dir.close();
+ super.tearDown();
+ }
+
+ protected Query tq( String text ){
+ return tq( 1F, text );
+ }
+
+ protected Query tq( float boost, String text ){
+ return tq( boost, F, text );
+ }
+
+ protected Query tq( String field, String text ){
+ return tq( 1F, field, text );
+ }
+
+ protected Query tq( float boost, String field, String text ){
+ Query query = new TermQuery( new Term( field, text ) );
+ query.setBoost( boost );
+ return query;
+ }
+
+ protected Query pqF( String... texts ){
+ return pqF( 1F, texts );
+ }
+
+ protected Query pqF( float boost, String... texts ){
+ return pqF( boost, 0, texts );
+ }
+
+ protected Query pqF( float boost, int slop, String... texts ){
+ return pq( boost, slop, F, texts );
+ }
+
+ protected Query pq( String field, String... texts ){
+ return pq( 1F, 0, field, texts );
+ }
+
+ protected Query pq( float boost, String field, String... texts ){
+ return pq( boost, 0, field, texts );
+ }
+
+ protected Query pq( float boost, int slop, String field, String... texts ){
+ PhraseQuery query = new PhraseQuery();
+ for( String text : texts ){
+ query.add( new Term( field, text ) );
+ }
+ query.setBoost( boost );
+ query.setSlop( slop );
+ return query;
+ }
+
+ protected Query dmq( Query... queries ){
+ return dmq( 0.0F, queries );
+ }
+
+ protected Query dmq( float tieBreakerMultiplier, Query... queries ){
+ DisjunctionMaxQuery query = new DisjunctionMaxQuery( tieBreakerMultiplier );
+ for( Query q : queries ){
+ query.add( q );
+ }
+ return query;
+ }
+
+ protected void assertCollectionQueries( Collection<Query> actual, Query... expected ){
+ assertEquals( expected.length, actual.size() );
+ for( Query query : expected ){
+ assertTrue( actual.contains( query ) );
+ }
+ }
+
+ static final class BigramAnalyzer extends Analyzer {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new BasicNGramTokenizer( reader );
+ }
+ }
+
+ static final class BasicNGramTokenizer extends Tokenizer {
+
+ public static final int DEFAULT_N_SIZE = 2;
+ public static final String DEFAULT_DELIMITERS = " \t\n.,";
+ private final int n;
+ private final String delimiters;
+ private int startTerm;
+ private int lenTerm;
+ private int startOffset;
+ private int nextStartOffset;
+ private int ch;
+ private String snippet;
+ private StringBuilder snippetBuffer;
+ private static final int BUFFER_SIZE = 4096;
+ private char[] charBuffer;
+ private int charBufferIndex;
+ private int charBufferLen;
+
+ public BasicNGramTokenizer( Reader in ){
+ this( in, DEFAULT_N_SIZE );
+ }
+
+ public BasicNGramTokenizer( Reader in, int n ){
+ this( in, n, DEFAULT_DELIMITERS );
+ }
+
+ public BasicNGramTokenizer( Reader in, String delimiters ){
+ this( in, DEFAULT_N_SIZE, delimiters );
+ }
+
+ public BasicNGramTokenizer( Reader in, int n, String delimiters ){
+ super(in);
+ this.n = n;
+ this.delimiters = delimiters;
+ startTerm = 0;
+ nextStartOffset = 0;
+ snippet = null;
+ snippetBuffer = new StringBuilder();
+ charBuffer = new char[BUFFER_SIZE];
+ charBufferIndex = BUFFER_SIZE;
+ charBufferLen = 0;
+ ch = 0;
+ }
+
+ CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ @Override
+ public boolean incrementToken() throws IOException {
+ if( !getNextPartialSnippet() )
+ return false;
+ clearAttributes();
+ termAtt.setEmpty().append(snippet, startTerm, startTerm + lenTerm);
+ offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm));
+ return true;
+ }
+
+ private int getFinalOffset() {
+ return nextStartOffset;
+ }
+
+ @Override
+ public final void end(){
+ offsetAtt.setOffset(getFinalOffset(),getFinalOffset());
+ }
+
+ protected boolean getNextPartialSnippet() throws IOException {
+ if( snippet != null && snippet.length() >= startTerm + 1 + n ){
+ startTerm++;
+ startOffset++;
+ lenTerm = n;
+ return true;
+ }
+ return getNextSnippet();
+ }
+
+ protected boolean getNextSnippet() throws IOException {
+ startTerm = 0;
+ startOffset = nextStartOffset;
+ snippetBuffer.delete( 0, snippetBuffer.length() );
+ while( true ){
+ if( ch != -1 )
+ ch = readCharFromBuffer();
+ if( ch == -1 ) break;
+ else if( !isDelimiter( ch ) )
+ snippetBuffer.append( (char)ch );
+ else if( snippetBuffer.length() > 0 )
+ break;
+ else
+ startOffset++;
+ }
+ if( snippetBuffer.length() == 0 )
+ return false;
+ snippet = snippetBuffer.toString();
+ lenTerm = snippet.length() >= n ? n : snippet.length();
+ return true;
+ }
+
+ protected int readCharFromBuffer() throws IOException {
+ if( charBufferIndex >= charBufferLen ){
+ charBufferLen = input.read( charBuffer );
+ if( charBufferLen == -1 ){
+ return -1;
+ }
+ charBufferIndex = 0;
+ }
+ int c = charBuffer[charBufferIndex++];
+ nextStartOffset++;
+ return c;
+ }
+
+ protected boolean isDelimiter( int c ){
+ return delimiters.indexOf( c ) >= 0;
+ }
+
+ @Override
+ public void reset( Reader input ) throws IOException {
+ super.reset( input );
+ reset();
+ }
+
+ @Override
+ public void reset() throws IOException {
+ startTerm = 0;
+ nextStartOffset = 0;
+ snippet = null;
+ snippetBuffer.setLength( 0 );
+ charBufferIndex = BUFFER_SIZE;
+ charBufferLen = 0;
+ ch = 0;
+ }
+ }
+
+ protected void make1d1fIndex( String value ) throws Exception {
+ make1dmfIndex( value );
+ }
+
+ protected void make1d1fIndexB( String value ) throws Exception {
+ make1dmfIndexB( value );
+ }
+
+ protected void make1dmfIndex( String... values ) throws Exception {
+ make1dmfIndex( analyzerW, values );
+ }
+
+ protected void make1dmfIndexB( String... values ) throws Exception {
+ make1dmfIndex( analyzerB, values );
+ }
+
+ // make 1 doc with multi valued field
+ protected void make1dmfIndex( Analyzer analyzer, String... values ) throws Exception {
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, analyzer).setOpenMode(OpenMode.CREATE));
+ Document doc = new Document();
+ for( String value: values )
+ doc.add( new Field( F, value, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
+ writer.addDocument( doc );
+ writer.close();
+ if (reader != null) reader.close();
+ reader = IndexReader.open( dir, true );
+ }
+
+ // make 1 doc with multi valued & not analyzed field
+ protected void make1dmfIndexNA( String... values ) throws Exception {
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, analyzerK).setOpenMode(OpenMode.CREATE));
+ Document doc = new Document();
+ for( String value: values )
+ doc.add( new Field( F, value, Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS ) );
+ writer.addDocument( doc );
+ writer.close();
+ if (reader != null) reader.close();
+ reader = IndexReader.open( dir, true );
+ }
+
+ protected void makeIndexShortMV() throws Exception {
+
+ // 0
+ // ""
+ // 1
+ // ""
+
+ // 234567
+ // "a b c"
+ // 0 1 2
+
+ // 8
+ // ""
+
+ // 111
+ // 9012
+ // "d e"
+ // 3 4
+ make1dmfIndex( shortMVValues );
+ }
+
+ protected void makeIndexLongMV() throws Exception {
+ // 11111111112222222222333333333344444444445555555555666666666677777777778888888888999
+ // 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ // Followings are the examples of customizable parameters and actual examples of customization:
+ // 0 1 2 3 4 5 6 7 8 9 10 11
+
+ // 1 2
+ // 999999900000000001111111111222222222233333333334444444444555555555566666666667777777777888888888899999999990000000000111111111122
+ // 345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901
+ // The most search engines use only one of these methods. Even the search engines that says they can use the both methods basically
+ // 12 13 (14) (15) 16 17 18 19 20 21 22 23 (24) (25) 26 27 28 29 30 31 32 33 34
+
+ make1dmfIndex( longMVValues );
+ }
+
+ protected void makeIndexLongMVB() throws Exception {
+ // "*" ... LF
+
+ // 1111111111222222222233333333334444444444555555
+ // 01234567890123456789012345678901234567890123456789012345
+ // *Lucene/Solr does not require such additional hardware.
+ // Lu 0 do 10 re 15 su 21 na 31
+ // uc 1 oe 11 eq 16 uc 22 al 32
+ // ce 2 es 12 qu 17 ch 23 ha 33
+ // en 3 no 13 ui 18 ad 24 ar 34
+ // ne 4 ot 14 ir 19 dd 25 rd 35
+ // e/ 5 re 20 di 26 dw 36
+ // /S 6 it 27 wa 37
+ // So 7 ti 28 ar 38
+ // ol 8 io 29 re 39
+ // lr 9 on 30
+
+ // 5555666666666677777777778888888888999999999
+ // 6789012345678901234567890123456789012345678
+ // *When you talk about processing speed, the
+ // Wh 40 ab 48 es 56 th 65
+ // he 41 bo 49 ss 57 he 66
+ // en 42 ou 50 si 58
+ // yo 43 ut 51 in 59
+ // ou 44 pr 52 ng 60
+ // ta 45 ro 53 sp 61
+ // al 46 oc 54 pe 62
+ // lk 47 ce 55 ee 63
+ // ed 64
+
+ make1dmfIndexB( biMVValues );
+ }
+
+ protected void makeIndexStrMV() throws Exception {
+
+ // 0123
+ // "abc"
+
+ // 34567
+ // "defg"
+
+ // 111
+ // 789012
+ // "hijkl"
+ make1dmfIndexNA( strMVValues );
+ }
+}
fnp / pylucene.git / blobdiff