1 package org.apache.lucene.analysis.shingle;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.io.Reader;
23 import org.apache.lucene.analysis.Analyzer;
24 import org.apache.lucene.analysis.TokenStream;
25 import org.apache.lucene.analysis.standard.StandardAnalyzer;
26 import org.apache.lucene.util.Version;
29 * A ShingleAnalyzerWrapper wraps a {@link ShingleFilter} around another {@link Analyzer}.
31 * A shingle is another name for a token based n-gram.
34 public final class ShingleAnalyzerWrapper extends Analyzer {
36 private final Analyzer defaultAnalyzer;
37 private int maxShingleSize = ShingleFilter.DEFAULT_MAX_SHINGLE_SIZE;
38 private int minShingleSize = ShingleFilter.DEFAULT_MIN_SHINGLE_SIZE;
39 private String tokenSeparator = ShingleFilter.TOKEN_SEPARATOR;
40 private boolean outputUnigrams = true;
41 private boolean outputUnigramsIfNoShingles = false;
43 public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer) {
44 this(defaultAnalyzer, ShingleFilter.DEFAULT_MAX_SHINGLE_SIZE);
47 public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer, int maxShingleSize) {
48 this(defaultAnalyzer, ShingleFilter.DEFAULT_MIN_SHINGLE_SIZE, maxShingleSize);
51 public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer, int minShingleSize, int maxShingleSize) {
52 this(defaultAnalyzer, minShingleSize, maxShingleSize, ShingleFilter.TOKEN_SEPARATOR, true, false);
56 * Creates a new ShingleAnalyzerWrapper
58 * @param defaultAnalyzer Analyzer whose TokenStream is to be filtered
59 * @param minShingleSize Min shingle (token ngram) size
60 * @param maxShingleSize Max shingle size
61 * @param tokenSeparator Used to separate input stream tokens in output shingles
62 * @param outputUnigrams Whether or not the filter shall pass the original
63 * tokens to the output stream
64 * @param outputUnigramsIfNoShingles Overrides the behavior of outputUnigrams==false for those
65 * times when no shingles are available (because there are fewer than
66 * minShingleSize tokens in the input stream)?
67 * Note that if outputUnigrams==true, then unigrams are always output,
68 * regardless of whether any shingles are available.
70 public ShingleAnalyzerWrapper(
71 Analyzer defaultAnalyzer,
74 String tokenSeparator,
75 boolean outputUnigrams,
76 boolean outputUnigramsIfNoShingles) {
77 this.defaultAnalyzer = defaultAnalyzer;
79 if (maxShingleSize < 2) {
80 throw new IllegalArgumentException("Max shingle size must be >= 2");
82 this.maxShingleSize = maxShingleSize;
84 if (minShingleSize < 2) {
85 throw new IllegalArgumentException("Min shingle size must be >= 2");
87 if (minShingleSize > maxShingleSize) {
88 throw new IllegalArgumentException
89 ("Min shingle size must be <= max shingle size");
91 this.minShingleSize = minShingleSize;
93 this.tokenSeparator = (tokenSeparator == null ? "" : tokenSeparator);
94 this.outputUnigrams = outputUnigrams;
95 this.outputUnigramsIfNoShingles = outputUnigramsIfNoShingles;
99 * Wraps {@link StandardAnalyzer}.
101 public ShingleAnalyzerWrapper(Version matchVersion) {
102 this(matchVersion, ShingleFilter.DEFAULT_MIN_SHINGLE_SIZE, ShingleFilter.DEFAULT_MAX_SHINGLE_SIZE);
106 * Wraps {@link StandardAnalyzer}.
108 public ShingleAnalyzerWrapper(Version matchVersion, int minShingleSize, int maxShingleSize) {
109 this(new StandardAnalyzer(matchVersion), minShingleSize, maxShingleSize);
113 * The max shingle (token ngram) size
115 * @return The max shingle (token ngram) size
117 public int getMaxShingleSize() {
118 return maxShingleSize;
122 * Set the maximum size of output shingles (default: 2)
124 * @param maxShingleSize max shingle size
125 * @deprecated Setting maxShingleSize after Analyzer instantiation prevents reuse.
126 * Confgure maxShingleSize during construction.
129 public void setMaxShingleSize(int maxShingleSize) {
130 if (maxShingleSize < 2) {
131 throw new IllegalArgumentException("Max shingle size must be >= 2");
133 this.maxShingleSize = maxShingleSize;
137 * The min shingle (token ngram) size
139 * @return The min shingle (token ngram) size
141 public int getMinShingleSize() {
142 return minShingleSize;
146 * <p>Set the min shingle size (default: 2).
147 * <p>This method requires that the passed in minShingleSize is not greater
148 * than maxShingleSize, so make sure that maxShingleSize is set before
149 * calling this method.
151 * @param minShingleSize min size of output shingles
152 * @deprecated Setting minShingleSize after Analyzer instantiation prevents reuse.
153 * Confgure minShingleSize during construction.
156 public void setMinShingleSize(int minShingleSize) {
157 if (minShingleSize < 2) {
158 throw new IllegalArgumentException("Min shingle size must be >= 2");
160 if (minShingleSize > maxShingleSize) {
161 throw new IllegalArgumentException
162 ("Min shingle size must be <= max shingle size");
164 this.minShingleSize = minShingleSize;
167 public String getTokenSeparator() {
168 return tokenSeparator;
172 * Sets the string to use when joining adjacent tokens to form a shingle
173 * @param tokenSeparator used to separate input stream tokens in output shingles
174 * @deprecated Setting tokenSeparator after Analyzer instantiation prevents reuse.
175 * Confgure tokenSeparator during construction.
178 public void setTokenSeparator(String tokenSeparator) {
179 this.tokenSeparator = (tokenSeparator == null ? "" : tokenSeparator);
182 public boolean isOutputUnigrams() {
183 return outputUnigrams;
187 * Shall the filter pass the original tokens (the "unigrams") to the output
190 * @param outputUnigrams Whether or not the filter shall pass the original
191 * tokens to the output stream
192 * @deprecated Setting outputUnigrams after Analyzer instantiation prevents reuse.
193 * Confgure outputUnigrams during construction.
196 public void setOutputUnigrams(boolean outputUnigrams) {
197 this.outputUnigrams = outputUnigrams;
200 public boolean isOutputUnigramsIfNoShingles() {
201 return outputUnigramsIfNoShingles;
205 * <p>Shall we override the behavior of outputUnigrams==false for those
206 * times when no shingles are available (because there are fewer than
207 * minShingleSize tokens in the input stream)? (default: false.)
208 * <p>Note that if outputUnigrams==true, then unigrams are always output,
209 * regardless of whether any shingles are available.
211 * @param outputUnigramsIfNoShingles Whether or not to output a single
212 * unigram when no shingles are available.
213 * @deprecated Setting outputUnigramsIfNoShingles after Analyzer instantiation prevents reuse.
214 * Confgure outputUnigramsIfNoShingles during construction.
217 public void setOutputUnigramsIfNoShingles(boolean outputUnigramsIfNoShingles) {
218 this.outputUnigramsIfNoShingles = outputUnigramsIfNoShingles;
222 public TokenStream tokenStream(String fieldName, Reader reader) {
225 wrapped = defaultAnalyzer.reusableTokenStream(fieldName, reader);
226 } catch (IOException e) {
227 wrapped = defaultAnalyzer.tokenStream(fieldName, reader);
229 ShingleFilter filter = new ShingleFilter(wrapped, minShingleSize, maxShingleSize);
230 filter.setMinShingleSize(minShingleSize);
231 filter.setMaxShingleSize(maxShingleSize);
232 filter.setTokenSeparator(tokenSeparator);
233 filter.setOutputUnigrams(outputUnigrams);
234 filter.setOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
238 private class SavedStreams {
240 ShingleFilter shingle;
244 public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
245 SavedStreams streams = (SavedStreams) getPreviousTokenStream();
246 if (streams == null) {
247 streams = new SavedStreams();
248 streams.wrapped = defaultAnalyzer.reusableTokenStream(fieldName, reader);
249 streams.shingle = new ShingleFilter(streams.wrapped);
250 setPreviousTokenStream(streams);
252 TokenStream result = defaultAnalyzer.reusableTokenStream(fieldName, reader);
253 if (result != streams.wrapped) {
254 /* the wrapped analyzer did not, create a new shingle around the new one */
255 streams.wrapped = result;
256 streams.shingle = new ShingleFilter(streams.wrapped);
259 streams.shingle.setMaxShingleSize(maxShingleSize);
260 streams.shingle.setMinShingleSize(minShingleSize);
261 streams.shingle.setTokenSeparator(tokenSeparator);
262 streams.shingle.setOutputUnigrams(outputUnigrams);
263 streams.shingle.setOutputUnigramsIfNoShingles(outputUnigramsIfNoShingles);
264 return streams.shingle;