1 package org.apache.lucene.analysis;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
22 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
23 import org.apache.lucene.queryParser.QueryParser; // for javadoc
26 * Abstract base class for TokenFilters that may remove tokens.
27 * You have to implement {@link #accept} and return a boolean if the current
28 * token should be preserved. {@link #incrementToken} uses this method
29 * to decide if a token should be passed to the caller.
31 public abstract class FilteringTokenFilter extends TokenFilter {
33 private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
34 private boolean enablePositionIncrements; // no init needed, as ctor enforces setting value!
36 public FilteringTokenFilter(boolean enablePositionIncrements, TokenStream input){
38 this.enablePositionIncrements = enablePositionIncrements;
41 /** Override this method and return if the current input token should be returned by {@link #incrementToken}. */
42 protected abstract boolean accept() throws IOException;
45 public final boolean incrementToken() throws IOException {
46 if (enablePositionIncrements) {
47 int skippedPositions = 0;
48 while (input.incrementToken()) {
50 if (skippedPositions != 0) {
51 posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions);
55 skippedPositions += posIncrAtt.getPositionIncrement();
58 while (input.incrementToken()) {
64 // reached EOS -- return false
69 * @see #setEnablePositionIncrements(boolean)
71 public boolean getEnablePositionIncrements() {
72 return enablePositionIncrements;
76 * If <code>true</code>, this TokenFilter will preserve
77 * positions of the incoming tokens (ie, accumulate and
78 * set position increments of the removed tokens).
79 * Generally, <code>true</code> is best as it does not
80 * lose information (positions of the original tokens)
83 * <p> When set, when a token is stopped
84 * (omitted), the position increment of the following
85 * token is incremented.
87 * <p> <b>NOTE</b>: be sure to also
88 * set {@link QueryParser#setEnablePositionIncrements} if
89 * you use QueryParser to create queries.
91 public void setEnablePositionIncrements(boolean enable) {
92 this.enablePositionIncrements = enable;