1 package org.apache.lucene.analysis.shingle;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.io.Reader;
23 import org.apache.lucene.analysis.Analyzer;
24 import org.apache.lucene.analysis.TokenStream;
25 import org.apache.lucene.analysis.standard.StandardAnalyzer;
26 import org.apache.lucene.util.Version;
29 * A ShingleAnalyzerWrapper wraps a {@link ShingleFilter} around another {@link Analyzer}.
31 * A shingle is another name for a token based n-gram.
34 public final class ShingleAnalyzerWrapper extends Analyzer {
36 private final Analyzer defaultAnalyzer;
37 private int maxShingleSize = ShingleFilter.DEFAULT_MAX_SHINGLE_SIZE;
38 private int minShingleSize = ShingleFilter.DEFAULT_MIN_SHINGLE_SIZE;
39 private String tokenSeparator = ShingleFilter.TOKEN_SEPARATOR;
40 private boolean outputUnigrams = true;
41 private boolean outputUnigramsIfNoShingles = false;
43 public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer) {
45 this.defaultAnalyzer = defaultAnalyzer;
48 public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer, int maxShingleSize) {
49 this(defaultAnalyzer);
50 setMaxShingleSize(maxShingleSize);
53 public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer, int minShingleSize, int maxShingleSize) {
54 this(defaultAnalyzer);
55 setMaxShingleSize(maxShingleSize);
56 setMinShingleSize(minShingleSize);
60 * Wraps {@link StandardAnalyzer}.
62 public ShingleAnalyzerWrapper(Version matchVersion) {
64 this.defaultAnalyzer = new StandardAnalyzer(matchVersion);
68 * Wraps {@link StandardAnalyzer}.
70 public ShingleAnalyzerWrapper(Version matchVersion, int minShingleSize, int maxShingleSize) {
72 setMaxShingleSize(maxShingleSize);
73 setMinShingleSize(minShingleSize);
77 * The max shingle (token ngram) size
79 * @return The max shingle (token ngram) size
81 public int getMaxShingleSize() {
82 return maxShingleSize;
86 * Set the maximum size of output shingles (default: 2)
88 * @param maxShingleSize max shingle size
90 public void setMaxShingleSize(int maxShingleSize) {
91 if (maxShingleSize < 2) {
92 throw new IllegalArgumentException("Max shingle size must be >= 2");
94 this.maxShingleSize = maxShingleSize;
98 * The min shingle (token ngram) size
100 * @return The min shingle (token ngram) size
102 public int getMinShingleSize() {
103 return minShingleSize;
107 * <p>Set the min shingle size (default: 2).
108 * <p>This method requires that the passed in minShingleSize is not greater
109 * than maxShingleSize, so make sure that maxShingleSize is set before
110 * calling this method.
112 * @param minShingleSize min size of output shingles
114 public void setMinShingleSize(int minShingleSize) {
115 if (minShingleSize < 2) {
116 throw new IllegalArgumentException("Min shingle size must be >= 2");
118 if (minShingleSize > maxShingleSize) {
119 throw new IllegalArgumentException
120 ("Min shingle size must be <= max shingle size");
122 this.minShingleSize = minShingleSize;
125 public String getTokenSeparator() {
126 return tokenSeparator;
130 * Sets the string to use when joining adjacent tokens to form a shingle
131 * @param tokenSeparator used to separate input stream tokens in output shingles
133 public void setTokenSeparator(String tokenSeparator) {
134 this.tokenSeparator = (tokenSeparator == null ? "" : tokenSeparator);
137 public boolean isOutputUnigrams() {
138 return outputUnigrams;
142 * Shall the filter pass the original tokens (the "unigrams") to the output
145 * @param outputUnigrams Whether or not the filter shall pass the original
146 * tokens to the output stream
148 public void setOutputUnigrams(boolean outputUnigrams) {
149 this.outputUnigrams = outputUnigrams;
152 public boolean isOutputUnigramsIfNoShingles() {
153 return outputUnigramsIfNoShingles;
157 * <p>Shall we override the behavior of outputUnigrams==false for those
158 * times when no shingles are available (because there are fewer than
159 * minShingleSize tokens in the input stream)? (default: false.)
160 * <p>Note that if outputUnigrams==true, then unigrams are always output,
161 * regardless of whether any shingles are available.
163 * @param outputUnigramsIfNoShingles Whether or not to output a single
164 * unigram when no shingles are available.
166 public void setOutputUnigramsIfNoShingles(boolean outputUnigramsIfNoShingles) {
167 this.outputUnigramsIfNoShingles = outputUnigramsIfNoShingles;
171 public TokenStream tokenStream(String fieldName, Reader reader) {
174 wrapped = defaultAnalyzer.reusableTokenStream(fieldName, reader);
175 } catch (IOException e) {
176 wrapped = defaultAnalyzer.tokenStream(fieldName, reader);
178 ShingleFilter filter = new ShingleFilter(wrapped, minShingleSize, maxShingleSize);
179 filter.setMinShingleSize(minShingleSize);
180 filter.setMaxShingleSize(maxShingleSize);
181 filter.setTokenSeparator(tokenSeparator);
182 filter.setOutputUnigrams(outputUnigrams);
183 filter.setOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
187 private class SavedStreams {
189 ShingleFilter shingle;
193 public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
194 SavedStreams streams = (SavedStreams) getPreviousTokenStream();
195 if (streams == null) {
196 streams = new SavedStreams();
197 streams.wrapped = defaultAnalyzer.reusableTokenStream(fieldName, reader);
198 streams.shingle = new ShingleFilter(streams.wrapped);
199 setPreviousTokenStream(streams);
201 TokenStream result = defaultAnalyzer.reusableTokenStream(fieldName, reader);
202 if (result != streams.wrapped) {
203 /* the wrapped analyzer did not, create a new shingle around the new one */
204 streams.wrapped = result;
205 streams.shingle = new ShingleFilter(streams.wrapped);
208 streams.shingle.setMaxShingleSize(maxShingleSize);
209 streams.shingle.setMinShingleSize(minShingleSize);
210 streams.shingle.setTokenSeparator(tokenSeparator);
211 streams.shingle.setOutputUnigrams(outputUnigrams);
212 streams.shingle.setOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
213 return streams.shingle;