+++ /dev/null
-package org.apache.lucene.analysis.shingle;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.io.StringReader;
-
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
-import org.apache.lucene.analysis.tokenattributes.*;
-
-public class ShingleFilterTest extends BaseTokenStreamTestCase {
-
- public class TestTokenStream extends TokenStream {
-
- protected int index = 0;
- protected Token[] testToken;
-
- private CharTermAttribute termAtt;
- private OffsetAttribute offsetAtt;
- private PositionIncrementAttribute posIncrAtt;
- private TypeAttribute typeAtt;
-
- public TestTokenStream(Token[] testToken) {
- super();
- this.testToken = testToken;
- this.termAtt = addAttribute(CharTermAttribute.class);
- this.offsetAtt = addAttribute(OffsetAttribute.class);
- this.posIncrAtt = addAttribute(PositionIncrementAttribute.class);
- this.typeAtt = addAttribute(TypeAttribute.class);
- }
-
- @Override
- public final boolean incrementToken() throws IOException {
- clearAttributes();
- if (index < testToken.length) {
- Token t = testToken[index++];
- termAtt.copyBuffer(t.buffer(), 0, t.length());
- offsetAtt.setOffset(t.startOffset(), t.endOffset());
- posIncrAtt.setPositionIncrement(t.getPositionIncrement());
- typeAtt.setType(TypeAttributeImpl.DEFAULT_TYPE);
- return true;
- } else {
- return false;
- }
- }
- }
-
- public static final Token[] TEST_TOKEN = new Token[] {
- createToken("please", 0, 6),
- createToken("divide", 7, 13),
- createToken("this", 14, 18),
- createToken("sentence", 19, 27),
- createToken("into", 28, 32),
- createToken("shingles", 33, 39),
- };
-
- public static final int[] UNIGRAM_ONLY_POSITION_INCREMENTS = new int[] {
- 1, 1, 1, 1, 1, 1
- };
-
- public static final String[] UNIGRAM_ONLY_TYPES = new String[] {
- "word", "word", "word", "word", "word", "word"
- };
-
- public static Token[] testTokenWithHoles;
-
- public static final Token[] BI_GRAM_TOKENS = new Token[] {
- createToken("please", 0, 6),
- createToken("please divide", 0, 13),
- createToken("divide", 7, 13),
- createToken("divide this", 7, 18),
- createToken("this", 14, 18),
- createToken("this sentence", 14, 27),
- createToken("sentence", 19, 27),
- createToken("sentence into", 19, 32),
- createToken("into", 28, 32),
- createToken("into shingles", 28, 39),
- createToken("shingles", 33, 39),
- };
-
- public static final int[] BI_GRAM_POSITION_INCREMENTS = new int[] {
- 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
- };
-
- public static final String[] BI_GRAM_TYPES = new String[] {
- "word", "shingle", "word", "shingle", "word", "shingle", "word",
- "shingle", "word", "shingle", "word"
- };
-
- public static final Token[] BI_GRAM_TOKENS_WITH_HOLES = new Token[] {
- createToken("please", 0, 6),
- createToken("please divide", 0, 13),
- createToken("divide", 7, 13),
- createToken("divide _", 7, 19),
- createToken("_ sentence", 19, 27),
- createToken("sentence", 19, 27),
- createToken("sentence _", 19, 33),
- createToken("_ shingles", 33, 39),
- createToken("shingles", 33, 39),
- };
-
- public static final int[] BI_GRAM_POSITION_INCREMENTS_WITH_HOLES = new int[] {
- 1, 0, 1, 0, 1, 1, 0, 1, 1
- };
-
- private static final String[] BI_GRAM_TYPES_WITH_HOLES = {
- "word", "shingle",
- "word", "shingle", "shingle", "word", "shingle", "shingle", "word"
- };
-
- public static final Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS = new Token[] {
- createToken("please divide", 0, 13),
- createToken("divide this", 7, 18),
- createToken("this sentence", 14, 27),
- createToken("sentence into", 19, 32),
- createToken("into shingles", 28, 39),
- };
-
- public static final int[] BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS = new int[] {
- 1, 1, 1, 1, 1
- };
-
- public static final String[] BI_GRAM_TYPES_WITHOUT_UNIGRAMS = new String[] {
- "shingle", "shingle", "shingle", "shingle", "shingle"
- };
-
- public static final Token[] BI_GRAM_TOKENS_WITH_HOLES_WITHOUT_UNIGRAMS = new Token[] {
- createToken("please divide", 0, 13),
- createToken("divide _", 7, 19),
- createToken("_ sentence", 19, 27),
- createToken("sentence _", 19, 33),
- createToken("_ shingles", 33, 39),
- };
-
- public static final int[] BI_GRAM_POSITION_INCREMENTS_WITH_HOLES_WITHOUT_UNIGRAMS = new int[] {
- 1, 1, 1, 1, 1, 1
- };
-
-
- public static final Token[] TEST_SINGLE_TOKEN = new Token[] {
- createToken("please", 0, 6)
- };
-
- public static final Token[] SINGLE_TOKEN = new Token[] {
- createToken("please", 0, 6)
- };
-
- public static final int[] SINGLE_TOKEN_INCREMENTS = new int[] {
- 1
- };
-
- public static final String[] SINGLE_TOKEN_TYPES = new String[] {
- "word"
- };
-
- public static final Token[] EMPTY_TOKEN_ARRAY = new Token[] {
- };
-
- public static final int[] EMPTY_TOKEN_INCREMENTS_ARRAY = new int[] {
- };
-
- public static final String[] EMPTY_TOKEN_TYPES_ARRAY = new String[] {
- };
-
- public static final Token[] TRI_GRAM_TOKENS = new Token[] {
- createToken("please", 0, 6),
- createToken("please divide", 0, 13),
- createToken("please divide this", 0, 18),
- createToken("divide", 7, 13),
- createToken("divide this", 7, 18),
- createToken("divide this sentence", 7, 27),
- createToken("this", 14, 18),
- createToken("this sentence", 14, 27),
- createToken("this sentence into", 14, 32),
- createToken("sentence", 19, 27),
- createToken("sentence into", 19, 32),
- createToken("sentence into shingles", 19, 39),
- createToken("into", 28, 32),
- createToken("into shingles", 28, 39),
- createToken("shingles", 33, 39)
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS = new int[] {
- 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1
- };
-
- public static final String[] TRI_GRAM_TYPES = new String[] {
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle",
- "word"
- };
-
- public static final Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS = new Token[] {
- createToken("please divide", 0, 13),
- createToken("please divide this", 0, 18),
- createToken("divide this", 7, 18),
- createToken("divide this sentence", 7, 27),
- createToken("this sentence", 14, 27),
- createToken("this sentence into", 14, 32),
- createToken("sentence into", 19, 32),
- createToken("sentence into shingles", 19, 39),
- createToken("into shingles", 28, 39),
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS = new int[] {
- 1, 0, 1, 0, 1, 0, 1, 0, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS = new String[] {
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle",
- };
-
- public static final Token[] FOUR_GRAM_TOKENS = new Token[] {
- createToken("please", 0, 6),
- createToken("please divide", 0, 13),
- createToken("please divide this", 0, 18),
- createToken("please divide this sentence", 0, 27),
- createToken("divide", 7, 13),
- createToken("divide this", 7, 18),
- createToken("divide this sentence", 7, 27),
- createToken("divide this sentence into", 7, 32),
- createToken("this", 14, 18),
- createToken("this sentence", 14, 27),
- createToken("this sentence into", 14, 32),
- createToken("this sentence into shingles", 14, 39),
- createToken("sentence", 19, 27),
- createToken("sentence into", 19, 32),
- createToken("sentence into shingles", 19, 39),
- createToken("into", 28, 32),
- createToken("into shingles", 28, 39),
- createToken("shingles", 33, 39)
- };
-
- public static final int[] FOUR_GRAM_POSITION_INCREMENTS = new int[] {
- 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1
- };
-
- public static final String[] FOUR_GRAM_TYPES = new String[] {
- "word", "shingle", "shingle", "shingle",
- "word", "shingle", "shingle", "shingle",
- "word", "shingle", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle",
- "word"
- };
-
- public static final Token[] FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS = new Token[] {
- createToken("please divide", 0, 13),
- createToken("please divide this", 0, 18),
- createToken("please divide this sentence", 0, 27),
- createToken("divide this", 7, 18),
- createToken("divide this sentence", 7, 27),
- createToken("divide this sentence into", 7, 32),
- createToken("this sentence", 14, 27),
- createToken("this sentence into", 14, 32),
- createToken("this sentence into shingles", 14, 39),
- createToken("sentence into", 19, 32),
- createToken("sentence into shingles", 19, 39),
- createToken("into shingles", 28, 39),
- };
-
- public static final int[] FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS = new int[] {
- 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1
- };
-
- public static final String[] FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS = new String[] {
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
-
- };
-
- public static final Token[] TRI_GRAM_TOKENS_MIN_TRI_GRAM = new Token[] {
- createToken("please", 0, 6),
- createToken("please divide this", 0, 18),
- createToken("divide", 7, 13),
- createToken("divide this sentence", 7, 27),
- createToken("this", 14, 18),
- createToken("this sentence into", 14, 32),
- createToken("sentence", 19, 27),
- createToken("sentence into shingles", 19, 39),
- createToken("into", 28, 32),
- createToken("shingles", 33, 39)
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM = new int[] {
- 1, 0, 1, 0, 1, 0, 1, 0, 1, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_MIN_TRI_GRAM = new String[] {
- "word", "shingle",
- "word", "shingle",
- "word", "shingle",
- "word", "shingle",
- "word",
- "word"
- };
-
- public static final Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new Token[] {
- createToken("please divide this", 0, 18),
- createToken("divide this sentence", 7, 27),
- createToken("this sentence into", 14, 32),
- createToken("sentence into shingles", 19, 39)
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new int[] {
- 1, 1, 1, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new String[] {
- "shingle",
- "shingle",
- "shingle",
- "shingle"
- };
-
- public static final Token[] FOUR_GRAM_TOKENS_MIN_TRI_GRAM = new Token[] {
- createToken("please", 0, 6),
- createToken("please divide this", 0, 18),
- createToken("please divide this sentence", 0, 27),
- createToken("divide", 7, 13),
- createToken("divide this sentence", 7, 27),
- createToken("divide this sentence into", 7, 32),
- createToken("this", 14, 18),
- createToken("this sentence into", 14, 32),
- createToken("this sentence into shingles", 14, 39),
- createToken("sentence", 19, 27),
- createToken("sentence into shingles", 19, 39),
- createToken("into", 28, 32),
- createToken("shingles", 33, 39)
- };
-
- public static final int[] FOUR_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM = new int[] {
- 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 1
- };
-
- public static final String[] FOUR_GRAM_TYPES_MIN_TRI_GRAM = new String[] {
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle",
- "word",
- "word"
- };
-
- public static final Token[] FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new Token[] {
- createToken("please divide this", 0, 18),
- createToken("please divide this sentence", 0, 27),
- createToken("divide this sentence", 7, 27),
- createToken("divide this sentence into", 7, 32),
- createToken("this sentence into", 14, 32),
- createToken("this sentence into shingles", 14, 39),
- createToken("sentence into shingles", 19, 39),
- };
-
- public static final int[] FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new int[] {
- 1, 0, 1, 0, 1, 0, 1
- };
-
- public static final String[] FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM = new String[] {
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle"
- };
-
- public static final Token[] FOUR_GRAM_TOKENS_MIN_FOUR_GRAM = new Token[] {
- createToken("please", 0, 6),
- createToken("please divide this sentence", 0, 27),
- createToken("divide", 7, 13),
- createToken("divide this sentence into", 7, 32),
- createToken("this", 14, 18),
- createToken("this sentence into shingles", 14, 39),
- createToken("sentence", 19, 27),
- createToken("into", 28, 32),
- createToken("shingles", 33, 39)
- };
-
- public static final int[] FOUR_GRAM_POSITION_INCREMENTS_MIN_FOUR_GRAM = new int[] {
- 1, 0, 1, 0, 1, 0, 1, 1, 1
- };
-
- public static final String[] FOUR_GRAM_TYPES_MIN_FOUR_GRAM = new String[] {
- "word", "shingle",
- "word", "shingle",
- "word", "shingle",
- "word",
- "word",
- "word"
- };
-
- public static final Token[] FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM = new Token[] {
- createToken("please divide this sentence", 0, 27),
- createToken("divide this sentence into", 7, 32),
- createToken("this sentence into shingles", 14, 39),
- };
-
- public static final int[] FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM = new int[] {
- 1, 1, 1
- };
-
- public static final String[] FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM = new String[] {
- "shingle",
- "shingle",
- "shingle"
- };
-
- public static final Token[] BI_GRAM_TOKENS_NO_SEPARATOR = new Token[] {
- createToken("please", 0, 6),
- createToken("pleasedivide", 0, 13),
- createToken("divide", 7, 13),
- createToken("dividethis", 7, 18),
- createToken("this", 14, 18),
- createToken("thissentence", 14, 27),
- createToken("sentence", 19, 27),
- createToken("sentenceinto", 19, 32),
- createToken("into", 28, 32),
- createToken("intoshingles", 28, 39),
- createToken("shingles", 33, 39),
- };
-
- public static final int[] BI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR = new int[] {
- 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
- };
-
- public static final String[] BI_GRAM_TYPES_NO_SEPARATOR = new String[] {
- "word", "shingle", "word", "shingle", "word", "shingle", "word",
- "shingle", "word", "shingle", "word"
- };
-
- public static final Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR = new Token[] {
- createToken("pleasedivide", 0, 13),
- createToken("dividethis", 7, 18),
- createToken("thissentence", 14, 27),
- createToken("sentenceinto", 19, 32),
- createToken("intoshingles", 28, 39),
- };
-
- public static final int[] BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR = new int[] {
- 1, 1, 1, 1, 1
- };
-
- public static final String[] BI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR = new String[] {
- "shingle", "shingle", "shingle", "shingle", "shingle"
- };
-
- public static final Token[] TRI_GRAM_TOKENS_NO_SEPARATOR = new Token[] {
- createToken("please", 0, 6),
- createToken("pleasedivide", 0, 13),
- createToken("pleasedividethis", 0, 18),
- createToken("divide", 7, 13),
- createToken("dividethis", 7, 18),
- createToken("dividethissentence", 7, 27),
- createToken("this", 14, 18),
- createToken("thissentence", 14, 27),
- createToken("thissentenceinto", 14, 32),
- createToken("sentence", 19, 27),
- createToken("sentenceinto", 19, 32),
- createToken("sentenceintoshingles", 19, 39),
- createToken("into", 28, 32),
- createToken("intoshingles", 28, 39),
- createToken("shingles", 33, 39)
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR = new int[] {
- 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_NO_SEPARATOR = new String[] {
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle",
- "word"
- };
-
- public static final Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR = new Token[] {
- createToken("pleasedivide", 0, 13),
- createToken("pleasedividethis", 0, 18),
- createToken("dividethis", 7, 18),
- createToken("dividethissentence", 7, 27),
- createToken("thissentence", 14, 27),
- createToken("thissentenceinto", 14, 32),
- createToken("sentenceinto", 19, 32),
- createToken("sentenceintoshingles", 19, 39),
- createToken("intoshingles", 28, 39),
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR = new int[] {
- 1, 0, 1, 0, 1, 0, 1, 0, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR = new String[] {
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle",
- };
-
- public static final Token[] BI_GRAM_TOKENS_ALT_SEPARATOR = new Token[] {
- createToken("please", 0, 6),
- createToken("please<SEP>divide", 0, 13),
- createToken("divide", 7, 13),
- createToken("divide<SEP>this", 7, 18),
- createToken("this", 14, 18),
- createToken("this<SEP>sentence", 14, 27),
- createToken("sentence", 19, 27),
- createToken("sentence<SEP>into", 19, 32),
- createToken("into", 28, 32),
- createToken("into<SEP>shingles", 28, 39),
- createToken("shingles", 33, 39),
- };
-
- public static final int[] BI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR = new int[] {
- 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1
- };
-
- public static final String[] BI_GRAM_TYPES_ALT_SEPARATOR = new String[] {
- "word", "shingle", "word", "shingle", "word", "shingle", "word",
- "shingle", "word", "shingle", "word"
- };
-
- public static final Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new Token[] {
- createToken("please<SEP>divide", 0, 13),
- createToken("divide<SEP>this", 7, 18),
- createToken("this<SEP>sentence", 14, 27),
- createToken("sentence<SEP>into", 19, 32),
- createToken("into<SEP>shingles", 28, 39),
- };
-
- public static final int[] BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new int[] {
- 1, 1, 1, 1, 1
- };
-
- public static final String[] BI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new String[] {
- "shingle", "shingle", "shingle", "shingle", "shingle"
- };
-
- public static final Token[] TRI_GRAM_TOKENS_ALT_SEPARATOR = new Token[] {
- createToken("please", 0, 6),
- createToken("please<SEP>divide", 0, 13),
- createToken("please<SEP>divide<SEP>this", 0, 18),
- createToken("divide", 7, 13),
- createToken("divide<SEP>this", 7, 18),
- createToken("divide<SEP>this<SEP>sentence", 7, 27),
- createToken("this", 14, 18),
- createToken("this<SEP>sentence", 14, 27),
- createToken("this<SEP>sentence<SEP>into", 14, 32),
- createToken("sentence", 19, 27),
- createToken("sentence<SEP>into", 19, 32),
- createToken("sentence<SEP>into<SEP>shingles", 19, 39),
- createToken("into", 28, 32),
- createToken("into<SEP>shingles", 28, 39),
- createToken("shingles", 33, 39)
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR = new int[] {
- 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_ALT_SEPARATOR = new String[] {
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle",
- "word"
- };
-
- public static final Token[] TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new Token[] {
- createToken("please<SEP>divide", 0, 13),
- createToken("please<SEP>divide<SEP>this", 0, 18),
- createToken("divide<SEP>this", 7, 18),
- createToken("divide<SEP>this<SEP>sentence", 7, 27),
- createToken("this<SEP>sentence", 14, 27),
- createToken("this<SEP>sentence<SEP>into", 14, 32),
- createToken("sentence<SEP>into", 19, 32),
- createToken("sentence<SEP>into<SEP>shingles", 19, 39),
- createToken("into<SEP>shingles", 28, 39),
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new int[] {
- 1, 0, 1, 0, 1, 0, 1, 0, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR = new String[] {
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle",
- };
-
- public static final Token[] TRI_GRAM_TOKENS_NULL_SEPARATOR = new Token[] {
- createToken("please", 0, 6),
- createToken("pleasedivide", 0, 13),
- createToken("pleasedividethis", 0, 18),
- createToken("divide", 7, 13),
- createToken("dividethis", 7, 18),
- createToken("dividethissentence", 7, 27),
- createToken("this", 14, 18),
- createToken("thissentence", 14, 27),
- createToken("thissentenceinto", 14, 32),
- createToken("sentence", 19, 27),
- createToken("sentenceinto", 19, 32),
- createToken("sentenceintoshingles", 19, 39),
- createToken("into", 28, 32),
- createToken("intoshingles", 28, 39),
- createToken("shingles", 33, 39)
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_NULL_SEPARATOR = new int[] {
- 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_NULL_SEPARATOR = new String[] {
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle",
- "word"
- };
-
- public static final Token[] TEST_TOKEN_POS_INCR_EQUAL_TO_N = new Token[] {
- createToken("please", 0, 6),
- createToken("divide", 7, 13),
- createToken("this", 14, 18),
- createToken("sentence", 29, 37, 3),
- createToken("into", 38, 42),
- createToken("shingles", 43, 49),
- };
-
- public static final Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N = new Token[] {
- createToken("please", 0, 6),
- createToken("please divide", 0, 13),
- createToken("please divide this", 0, 18),
- createToken("divide", 7, 13),
- createToken("divide this", 7, 18),
- createToken("divide this _", 7, 29),
- createToken("this", 14, 18),
- createToken("this _", 14, 29),
- createToken("this _ _", 14, 29),
- createToken("_ _ sentence", 29, 37),
- createToken("_ sentence", 29, 37),
- createToken("_ sentence into", 29, 42),
- createToken("sentence", 29, 37),
- createToken("sentence into", 29, 42),
- createToken("sentence into shingles", 29, 49),
- createToken("into", 38, 42),
- createToken("into shingles", 38, 49),
- createToken("shingles", 43, 49)
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N = new int[] {
- 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N = new String[] {
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "shingle", "shingle", "shingle", "word", "shingle", "shingle",
- "word", "shingle",
- "word"
- };
-
- public static final Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new Token[] {
- createToken("please divide", 0, 13),
- createToken("please divide this", 0, 18),
- createToken("divide this", 7, 18),
- createToken("divide this _", 7, 29),
- createToken("this _", 14, 29),
- createToken("this _ _", 14, 29),
- createToken("_ _ sentence", 29, 37),
- createToken("_ sentence", 29, 37),
- createToken("_ sentence into", 29, 42),
- createToken("sentence into", 29, 42),
- createToken("sentence into shingles", 29, 49),
- createToken("into shingles", 38, 49),
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new int[] {
- 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new String[] {
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle", "shingle",
- "shingle", "shingle",
- "shingle",
- };
-
- public static final Token[] TEST_TOKEN_POS_INCR_GREATER_THAN_N = new Token[] {
- createToken("please", 0, 6),
- createToken("divide", 57, 63, 8),
- createToken("this", 64, 68),
- createToken("sentence", 69, 77),
- createToken("into", 78, 82),
- createToken("shingles", 83, 89),
- };
-
- public static final Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N = new Token[] {
- createToken("please", 0, 6),
- createToken("please _", 0, 57),
- createToken("please _ _", 0, 57),
- createToken("_ _ divide", 57, 63),
- createToken("_ divide", 57, 63),
- createToken("_ divide this", 57, 68),
- createToken("divide", 57, 63),
- createToken("divide this", 57, 68),
- createToken("divide this sentence", 57, 77),
- createToken("this", 64, 68),
- createToken("this sentence", 64, 77),
- createToken("this sentence into", 64, 82),
- createToken("sentence", 69, 77),
- createToken("sentence into", 69, 82),
- createToken("sentence into shingles", 69, 89),
- createToken("into", 78, 82),
- createToken("into shingles", 78, 89),
- createToken("shingles", 83, 89)
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N = new int[] {
- 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1
- };
- public static final String[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N = new String[] {
- "word", "shingle", "shingle",
- "shingle",
- "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle", "shingle",
- "word", "shingle",
- "word"
- };
-
- public static final Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new Token[] {
- createToken("please _", 0, 57),
- createToken("please _ _", 0, 57),
- createToken("_ _ divide", 57, 63),
- createToken("_ divide", 57, 63),
- createToken("_ divide this", 57, 68),
- createToken("divide this", 57, 68),
- createToken("divide this sentence", 57, 77),
- createToken("this sentence", 64, 77),
- createToken("this sentence into", 64, 82),
- createToken("sentence into", 69, 82),
- createToken("sentence into shingles", 69, 89),
- createToken("into shingles", 78, 89),
- };
-
- public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new int[] {
- 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1
- };
-
- public static final String[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new String[] {
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle",
- "shingle", "shingle", "shingle", "shingle", "shingle",
- "shingle",
- };
-
- @Override
- public void setUp() throws Exception {
- super.setUp();
- testTokenWithHoles = new Token[] {
- createToken("please", 0, 6),
- createToken("divide", 7, 13),
- createToken("sentence", 19, 27, 2),
- createToken("shingles", 33, 39, 2),
- };
- }
-
- /*
- * Class under test for void ShingleFilter(TokenStream, int)
- */
- public void testBiGramFilter() throws IOException {
- this.shingleFilterTest(2, TEST_TOKEN, BI_GRAM_TOKENS,
- BI_GRAM_POSITION_INCREMENTS, BI_GRAM_TYPES,
- true);
- }
-
- public void testBiGramFilterWithHoles() throws IOException {
- this.shingleFilterTest(2, testTokenWithHoles, BI_GRAM_TOKENS_WITH_HOLES,
- BI_GRAM_POSITION_INCREMENTS_WITH_HOLES,
- BI_GRAM_TYPES_WITH_HOLES,
- true);
- }
-
- public void testBiGramFilterWithoutUnigrams() throws IOException {
- this.shingleFilterTest(2, TEST_TOKEN, BI_GRAM_TOKENS_WITHOUT_UNIGRAMS,
- BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, BI_GRAM_TYPES_WITHOUT_UNIGRAMS,
- false);
- }
-
- public void testBiGramFilterWithHolesWithoutUnigrams() throws IOException {
- this.shingleFilterTest(2, testTokenWithHoles, BI_GRAM_TOKENS_WITH_HOLES_WITHOUT_UNIGRAMS,
- BI_GRAM_POSITION_INCREMENTS_WITH_HOLES_WITHOUT_UNIGRAMS, BI_GRAM_TYPES_WITHOUT_UNIGRAMS,
- false);
- }
-
- public void testBiGramFilterWithSingleToken() throws IOException {
- this.shingleFilterTest(2, TEST_SINGLE_TOKEN, SINGLE_TOKEN,
- SINGLE_TOKEN_INCREMENTS, SINGLE_TOKEN_TYPES,
- true);
- }
-
- public void testBiGramFilterWithSingleTokenWithoutUnigrams() throws IOException {
- this.shingleFilterTest(2, TEST_SINGLE_TOKEN, EMPTY_TOKEN_ARRAY,
- EMPTY_TOKEN_INCREMENTS_ARRAY, EMPTY_TOKEN_TYPES_ARRAY,
- false);
- }
-
- public void testBiGramFilterWithEmptyTokenStream() throws IOException {
- this.shingleFilterTest(2, EMPTY_TOKEN_ARRAY, EMPTY_TOKEN_ARRAY,
- EMPTY_TOKEN_INCREMENTS_ARRAY, EMPTY_TOKEN_TYPES_ARRAY,
- true);
- }
-
- public void testBiGramFilterWithEmptyTokenStreamWithoutUnigrams() throws IOException {
- this.shingleFilterTest(2, EMPTY_TOKEN_ARRAY, EMPTY_TOKEN_ARRAY,
- EMPTY_TOKEN_INCREMENTS_ARRAY, EMPTY_TOKEN_TYPES_ARRAY,
- false);
- }
-
- public void testTriGramFilter() throws IOException {
- this.shingleFilterTest(3, TEST_TOKEN, TRI_GRAM_TOKENS,
- TRI_GRAM_POSITION_INCREMENTS, TRI_GRAM_TYPES,
- true);
- }
-
- public void testTriGramFilterWithoutUnigrams() throws IOException {
- this.shingleFilterTest(3, TEST_TOKEN, TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS,
- TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, TRI_GRAM_TYPES_WITHOUT_UNIGRAMS,
- false);
- }
-
- public void testFourGramFilter() throws IOException {
- this.shingleFilterTest(4, TEST_TOKEN, FOUR_GRAM_TOKENS,
- FOUR_GRAM_POSITION_INCREMENTS, FOUR_GRAM_TYPES,
- true);
- }
-
- public void testFourGramFilterWithoutUnigrams() throws IOException {
- this.shingleFilterTest(4, TEST_TOKEN, FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS,
- FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS,
- FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS, false);
- }
-
-
- public void testTriGramFilterMinTriGram() throws IOException {
- this.shingleFilterTest(3, 3, TEST_TOKEN, TRI_GRAM_TOKENS_MIN_TRI_GRAM,
- TRI_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM,
- TRI_GRAM_TYPES_MIN_TRI_GRAM,
- true);
- }
-
- public void testTriGramFilterWithoutUnigramsMinTriGram() throws IOException {
- this.shingleFilterTest(3, 3, TEST_TOKEN,
- TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM,
- TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM,
- TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM,
- false);
- }
-
- public void testFourGramFilterMinTriGram() throws IOException {
- this.shingleFilterTest(3, 4, TEST_TOKEN, FOUR_GRAM_TOKENS_MIN_TRI_GRAM,
- FOUR_GRAM_POSITION_INCREMENTS_MIN_TRI_GRAM,
- FOUR_GRAM_TYPES_MIN_TRI_GRAM,
- true);
- }
-
- public void testFourGramFilterWithoutUnigramsMinTriGram() throws IOException {
- this.shingleFilterTest(3, 4, TEST_TOKEN,
- FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM,
- FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_TRI_GRAM,
- FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_TRI_GRAM, false);
- }
-
- public void testFourGramFilterMinFourGram() throws IOException {
- this.shingleFilterTest(4, 4, TEST_TOKEN, FOUR_GRAM_TOKENS_MIN_FOUR_GRAM,
- FOUR_GRAM_POSITION_INCREMENTS_MIN_FOUR_GRAM,
- FOUR_GRAM_TYPES_MIN_FOUR_GRAM,
- true);
- }
-
- public void testFourGramFilterWithoutUnigramsMinFourGram() throws IOException {
- this.shingleFilterTest(4, 4, TEST_TOKEN,
- FOUR_GRAM_TOKENS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM,
- FOUR_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM,
- FOUR_GRAM_TYPES_WITHOUT_UNIGRAMS_MIN_FOUR_GRAM, false);
- }
-
- public void testBiGramFilterNoSeparator() throws IOException {
- this.shingleFilterTest("", 2, 2, TEST_TOKEN, BI_GRAM_TOKENS_NO_SEPARATOR,
- BI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR,
- BI_GRAM_TYPES_NO_SEPARATOR, true);
- }
-
- public void testBiGramFilterWithoutUnigramsNoSeparator() throws IOException {
- this.shingleFilterTest("", 2, 2, TEST_TOKEN,
- BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR,
- BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR,
- BI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR,
- false);
- }
- public void testTriGramFilterNoSeparator() throws IOException {
- this.shingleFilterTest("", 2, 3, TEST_TOKEN, TRI_GRAM_TOKENS_NO_SEPARATOR,
- TRI_GRAM_POSITION_INCREMENTS_NO_SEPARATOR,
- TRI_GRAM_TYPES_NO_SEPARATOR, true);
- }
-
- public void testTriGramFilterWithoutUnigramsNoSeparator() throws IOException {
- this.shingleFilterTest("", 2, 3, TEST_TOKEN,
- TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_NO_SEPARATOR,
- TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_NO_SEPARATOR,
- TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_NO_SEPARATOR, false);
- }
-
- public void testBiGramFilterAltSeparator() throws IOException {
- this.shingleFilterTest("<SEP>", 2, 2, TEST_TOKEN, BI_GRAM_TOKENS_ALT_SEPARATOR,
- BI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR,
- BI_GRAM_TYPES_ALT_SEPARATOR, true);
- }
-
- public void testBiGramFilterWithoutUnigramsAltSeparator() throws IOException {
- this.shingleFilterTest("<SEP>", 2, 2, TEST_TOKEN,
- BI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR,
- BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR,
- BI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR,
- false);
- }
- public void testTriGramFilterAltSeparator() throws IOException {
- this.shingleFilterTest("<SEP>", 2, 3, TEST_TOKEN, TRI_GRAM_TOKENS_ALT_SEPARATOR,
- TRI_GRAM_POSITION_INCREMENTS_ALT_SEPARATOR,
- TRI_GRAM_TYPES_ALT_SEPARATOR, true);
- }
-
- public void testTriGramFilterWithoutUnigramsAltSeparator() throws IOException {
- this.shingleFilterTest("<SEP>", 2, 3, TEST_TOKEN,
- TRI_GRAM_TOKENS_WITHOUT_UNIGRAMS_ALT_SEPARATOR,
- TRI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS_ALT_SEPARATOR,
- TRI_GRAM_TYPES_WITHOUT_UNIGRAMS_ALT_SEPARATOR, false);
- }
-
- public void testTriGramFilterNullSeparator() throws IOException {
- this.shingleFilterTest(null, 2, 3, TEST_TOKEN, TRI_GRAM_TOKENS_NULL_SEPARATOR,
- TRI_GRAM_POSITION_INCREMENTS_NULL_SEPARATOR,
- TRI_GRAM_TYPES_NULL_SEPARATOR, true);
- }
-
- public void testPositionIncrementEqualToN() throws IOException {
- this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_EQUAL_TO_N, TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N,
- TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N,
- TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N, true);
- }
-
- public void testPositionIncrementEqualToNWithoutUnigrams() throws IOException {
- this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_EQUAL_TO_N, TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS,
- TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS,
- TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, false);
- }
-
-
- public void testPositionIncrementGreaterThanN() throws IOException {
- this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_GREATER_THAN_N, TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N,
- TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N,
- TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N, true);
- }
-
- public void testPositionIncrementGreaterThanNWithoutUnigrams() throws IOException {
- this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_GREATER_THAN_N, TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS,
- TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS,
- TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, false);
- }
-
- public void testReset() throws Exception {
- Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("please divide this sentence"));
- TokenStream filter = new ShingleFilter(wsTokenizer, 2);
- assertTokenStreamContents(filter,
- new String[]{"please","please divide","divide","divide this","this","this sentence","sentence"},
- new int[]{0,0,7,7,14,14,19}, new int[]{6,13,13,18,18,27,27},
- new String[]{TypeAttributeImpl.DEFAULT_TYPE,"shingle",TypeAttributeImpl.DEFAULT_TYPE,"shingle",TypeAttributeImpl.DEFAULT_TYPE,"shingle",TypeAttributeImpl.DEFAULT_TYPE},
- new int[]{1,0,1,0,1,0,1}
- );
- wsTokenizer.reset(new StringReader("please divide this sentence"));
- assertTokenStreamContents(filter,
- new String[]{"please","please divide","divide","divide this","this","this sentence","sentence"},
- new int[]{0,0,7,7,14,14,19}, new int[]{6,13,13,18,18,27,27},
- new String[]{TypeAttributeImpl.DEFAULT_TYPE,"shingle",TypeAttributeImpl.DEFAULT_TYPE,"shingle",TypeAttributeImpl.DEFAULT_TYPE,"shingle",TypeAttributeImpl.DEFAULT_TYPE},
- new int[]{1,0,1,0,1,0,1}
- );
- }
-
- public void testOutputUnigramsIfNoShinglesSingleTokenCase() throws IOException {
- // Single token input with outputUnigrams==false is the primary case where
- // enabling this option should alter program behavior.
- this.shingleFilterTest(2, 2, TEST_SINGLE_TOKEN, SINGLE_TOKEN,
- SINGLE_TOKEN_INCREMENTS, SINGLE_TOKEN_TYPES,
- false, true);
- }
-
- public void testOutputUnigramsIfNoShinglesWithSimpleBigram() throws IOException {
- // Here we expect the same result as with testBiGramFilter().
- this.shingleFilterTest(2, 2, TEST_TOKEN, BI_GRAM_TOKENS,
- BI_GRAM_POSITION_INCREMENTS, BI_GRAM_TYPES,
- true, true);
- }
-
- public void testOutputUnigramsIfNoShinglesWithSimpleUnigramlessBigram() throws IOException {
- // Here we expect the same result as with testBiGramFilterWithoutUnigrams().
- this.shingleFilterTest(2, 2, TEST_TOKEN, BI_GRAM_TOKENS_WITHOUT_UNIGRAMS,
- BI_GRAM_POSITION_INCREMENTS_WITHOUT_UNIGRAMS, BI_GRAM_TYPES_WITHOUT_UNIGRAMS,
- false, true);
- }
-
- public void testOutputUnigramsIfNoShinglesWithMultipleInputTokens() throws IOException {
- // Test when the minimum shingle size is greater than the number of input tokens
- this.shingleFilterTest(7, 7, TEST_TOKEN, TEST_TOKEN,
- UNIGRAM_ONLY_POSITION_INCREMENTS, UNIGRAM_ONLY_TYPES,
- false, true);
- }
-
- protected void shingleFilterTest(int maxSize, Token[] tokensToShingle, Token[] tokensToCompare,
- int[] positionIncrements, String[] types,
- boolean outputUnigrams)
- throws IOException {
-
- ShingleFilter filter = new ShingleFilter(new TestTokenStream(tokensToShingle), maxSize);
- filter.setOutputUnigrams(outputUnigrams);
- shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
- }
-
- protected void shingleFilterTest(int minSize, int maxSize, Token[] tokensToShingle,
- Token[] tokensToCompare, int[] positionIncrements,
- String[] types, boolean outputUnigrams)
- throws IOException {
- ShingleFilter filter
- = new ShingleFilter(new TestTokenStream(tokensToShingle), minSize, maxSize);
- filter.setOutputUnigrams(outputUnigrams);
- shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
- }
-
- protected void shingleFilterTest(int minSize, int maxSize, Token[] tokensToShingle,
- Token[] tokensToCompare, int[] positionIncrements,
- String[] types, boolean outputUnigrams,
- boolean outputUnigramsIfNoShingles)
- throws IOException {
- ShingleFilter filter
- = new ShingleFilter(new TestTokenStream(tokensToShingle), minSize, maxSize);
- filter.setOutputUnigrams(outputUnigrams);
- filter.setOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
- shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
- }
-
- protected void shingleFilterTest(String tokenSeparator, int minSize, int maxSize, Token[] tokensToShingle,
- Token[] tokensToCompare, int[] positionIncrements,
- String[] types, boolean outputUnigrams)
- throws IOException {
- ShingleFilter filter
- = new ShingleFilter(new TestTokenStream(tokensToShingle), minSize, maxSize);
- filter.setTokenSeparator(tokenSeparator);
- filter.setOutputUnigrams(outputUnigrams);
- shingleFilterTestCommon(filter, tokensToCompare, positionIncrements, types);
- }
-
- protected void shingleFilterTestCommon(ShingleFilter filter,
- Token[] tokensToCompare,
- int[] positionIncrements,
- String[] types)
- throws IOException {
- String text[] = new String[tokensToCompare.length];
- int startOffsets[] = new int[tokensToCompare.length];
- int endOffsets[] = new int[tokensToCompare.length];
-
- for (int i = 0; i < tokensToCompare.length; i++) {
- text[i] = new String(tokensToCompare[i].buffer(),0, tokensToCompare[i].length());
- startOffsets[i] = tokensToCompare[i].startOffset();
- endOffsets[i] = tokensToCompare[i].endOffset();
- }
-
- assertTokenStreamContents(filter, text, startOffsets, endOffsets, types, positionIncrements);
- }
-
- private static Token createToken(String term, int start, int offset) {
- return createToken(term, start, offset, 1);
- }
-
- private static Token createToken
- (String term, int start, int offset, int positionIncrement)
- {
- Token token = new Token(start, offset);
- token.copyBuffer(term.toCharArray(), 0, term.length());
- token.setPositionIncrement(positionIncrement);
- return token;
- }
-}
fnp / pylucene.git / blobdiff