--- /dev/null
+package org.apache.lucene.analysis.query;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.StopFilter;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.Version;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.*;
+
+/**
+ * An {@link Analyzer} used primarily at query time to wrap another analyzer and provide a layer of protection
+ * which prevents very common words from being passed into queries.
+ * <p>
+ * For very large indexes the cost
+ * of reading TermDocs for a very common word can be high. This analyzer was created after experience with
+ * a 38 million doc index which had a term in around 50% of docs and was causing TermQueries for
+ * this term to take 2 seconds.
+ * </p>
+ * <p>
+ * Use the various "addStopWords" methods in this class to automate the identification and addition of
+ * stop words found in an already existing index.
+ * </p>
+ */
+public final class QueryAutoStopWordAnalyzer extends Analyzer {
+
+ private final Analyzer delegate;
+ private final Map<String, Set<String>> stopWordsPerField = new HashMap<String, Set<String>>();
+ //The default maximum percentage (40%) of index documents which
+ //can contain a term, after which the term is considered to be a stop word.
+ public static final float defaultMaxDocFreqPercent = 0.4f;
+ private final Version matchVersion;
+
+ /**
+ * Initializes this analyzer with the Analyzer object that actually produces the tokens
+ *
+ * @param delegate The choice of {@link Analyzer} that is used to produce the token stream which needs filtering
+ * @deprecated Stopwords should be calculated at instantiation using one of the other constructors
+ */
+ @Deprecated
+ public QueryAutoStopWordAnalyzer(Version matchVersion, Analyzer delegate) {
+ this.delegate = delegate;
+ this.matchVersion = matchVersion;
+ }
+
+ /**
+ * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for all
+ * indexed fields from terms with a document frequency percentage greater than
+ * {@link #defaultMaxDocFreqPercent}
+ *
+ * @param matchVersion Version to be used in {@link StopFilter}
+ * @param delegate Analyzer whose TokenStream will be filtered
+ * @param indexReader IndexReader to identify the stopwords from
+ * @throws IOException Can be thrown while reading from the IndexReader
+ */
+ public QueryAutoStopWordAnalyzer(
+ Version matchVersion,
+ Analyzer delegate,
+ IndexReader indexReader) throws IOException {
+ this(matchVersion, delegate, indexReader, defaultMaxDocFreqPercent);
+ }
+
+ /**
+ * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for all
+ * indexed fields from terms with a document frequency greater than the given
+ * maxDocFreq
+ *
+ * @param matchVersion Version to be used in {@link StopFilter}
+ * @param delegate Analyzer whose TokenStream will be filtered
+ * @param indexReader IndexReader to identify the stopwords from
+ * @param maxDocFreq Document frequency terms should be above in order to be stopwords
+ * @throws IOException Can be thrown while reading from the IndexReader
+ */
+ public QueryAutoStopWordAnalyzer(
+ Version matchVersion,
+ Analyzer delegate,
+ IndexReader indexReader,
+ int maxDocFreq) throws IOException {
+ this(matchVersion, delegate, indexReader, indexReader.getFieldNames(IndexReader.FieldOption.INDEXED), maxDocFreq);
+ }
+
+ /**
+ * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for all
+ * indexed fields from terms with a document frequency percentage greater than
+ * the given maxPercentDocs
+ *
+ * @param matchVersion Version to be used in {@link StopFilter}
+ * @param delegate Analyzer whose TokenStream will be filtered
+ * @param indexReader IndexReader to identify the stopwords from
+ * @param maxPercentDocs The maximum percentage (between 0.0 and 1.0) of index documents which
+ * contain a term, after which the word is considered to be a stop word
+ * @throws IOException Can be thrown while reading from the IndexReader
+ */
+ public QueryAutoStopWordAnalyzer(
+ Version matchVersion,
+ Analyzer delegate,
+ IndexReader indexReader,
+ float maxPercentDocs) throws IOException {
+ this(matchVersion, delegate, indexReader, indexReader.getFieldNames(IndexReader.FieldOption.INDEXED), maxPercentDocs);
+ }
+
+ /**
+ * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for the
+ * given selection of fields from terms with a document frequency percentage
+ * greater than the given maxPercentDocs
+ *
+ * @param matchVersion Version to be used in {@link StopFilter}
+ * @param delegate Analyzer whose TokenStream will be filtered
+ * @param indexReader IndexReader to identify the stopwords from
+ * @param fields Selection of fields to calculate stopwords for
+ * @param maxPercentDocs The maximum percentage (between 0.0 and 1.0) of index documents which
+ * contain a term, after which the word is considered to be a stop word
+ * @throws IOException Can be thrown while reading from the IndexReader
+ */
+ public QueryAutoStopWordAnalyzer(
+ Version matchVersion,
+ Analyzer delegate,
+ IndexReader indexReader,
+ Collection<String> fields,
+ float maxPercentDocs) throws IOException {
+ this(matchVersion, delegate, indexReader, fields, (int) (indexReader.numDocs() * maxPercentDocs));
+ }
+
+ /**
+ * Creates a new QueryAutoStopWordAnalyzer with stopwords calculated for the
+ * given selection of fields from terms with a document frequency greater than
+ * the given maxDocFreq
+ *
+ * @param matchVersion Version to be used in {@link StopFilter}
+ * @param delegate Analyzer whose TokenStream will be filtered
+ * @param indexReader IndexReader to identify the stopwords from
+ * @param fields Selection of fields to calculate stopwords for
+ * @param maxDocFreq Document frequency terms should be above in order to be stopwords
+ * @throws IOException Can be thrown while reading from the IndexReader
+ */
+ public QueryAutoStopWordAnalyzer(
+ Version matchVersion,
+ Analyzer delegate,
+ IndexReader indexReader,
+ Collection<String> fields,
+ int maxDocFreq) throws IOException {
+ this.matchVersion = matchVersion;
+ this.delegate = delegate;
+
+ for (String field : fields) {
+ Set<String> stopWords = new HashSet<String>();
+ String internedFieldName = StringHelper.intern(field);
+ TermEnum te = indexReader.terms(new Term(field));
+ Term term = te.term();
+ while (term != null) {
+ if (term.field() != internedFieldName) {
+ break;
+ }
+ if (te.docFreq() > maxDocFreq) {
+ stopWords.add(term.text());
+ }
+ if (!te.next()) {
+ break;
+ }
+ term = te.term();
+ }
+ stopWordsPerField.put(field, stopWords);
+ }
+ }
+
+ /**
+ * Automatically adds stop words for all fields with terms exceeding the defaultMaxDocFreqPercent
+ *
+ * @param reader The {@link IndexReader} which will be consulted to identify potential stop words that
+ * exceed the required document frequency
+ * @return The number of stop words identified.
+ * @throws IOException
+ * @deprecated Stopwords should be calculated at instantiation using
+ * {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader)}
+ */
+ @Deprecated
+ public int addStopWords(IndexReader reader) throws IOException {
+ return addStopWords(reader, defaultMaxDocFreqPercent);
+ }
+
+ /**
+ * Automatically adds stop words for all fields with terms exceeding the maxDocFreqPercent
+ *
+ * @param reader The {@link IndexReader} which will be consulted to identify potential stop words that
+ * exceed the required document frequency
+ * @param maxDocFreq The maximum number of index documents which can contain a term, after which
+ * the term is considered to be a stop word
+ * @return The number of stop words identified.
+ * @throws IOException
+ * @deprecated Stopwords should be calculated at instantiation using
+ * {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader, int)}
+ */
+ @Deprecated
+ public int addStopWords(IndexReader reader, int maxDocFreq) throws IOException {
+ int numStopWords = 0;
+ Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
+ for (Iterator<String> iter = fieldNames.iterator(); iter.hasNext();) {
+ String fieldName = iter.next();
+ numStopWords += addStopWords(reader, fieldName, maxDocFreq);
+ }
+ return numStopWords;
+ }
+
+ /**
+ * Automatically adds stop words for all fields with terms exceeding the maxDocFreqPercent
+ *
+ * @param reader The {@link IndexReader} which will be consulted to identify potential stop words that
+ * exceed the required document frequency
+ * @param maxPercentDocs The maximum percentage (between 0.0 and 1.0) of index documents which
+ * contain a term, after which the word is considered to be a stop word.
+ * @return The number of stop words identified.
+ * @throws IOException
+ * @deprecated Stowords should be calculated at instantiation using
+ * {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader, float)}
+ */
+ @Deprecated
+ public int addStopWords(IndexReader reader, float maxPercentDocs) throws IOException {
+ int numStopWords = 0;
+ Collection<String> fieldNames = reader.getFieldNames(IndexReader.FieldOption.INDEXED);
+ for (Iterator<String> iter = fieldNames.iterator(); iter.hasNext();) {
+ String fieldName = iter.next();
+ numStopWords += addStopWords(reader, fieldName, maxPercentDocs);
+ }
+ return numStopWords;
+ }
+
+ /**
+ * Automatically adds stop words for the given field with terms exceeding the maxPercentDocs
+ *
+ * @param reader The {@link IndexReader} which will be consulted to identify potential stop words that
+ * exceed the required document frequency
+ * @param fieldName The field for which stopwords will be added
+ * @param maxPercentDocs The maximum percentage (between 0.0 and 1.0) of index documents which
+ * contain a term, after which the word is considered to be a stop word.
+ * @return The number of stop words identified.
+ * @throws IOException
+ * @deprecated Stowords should be calculated at instantiation using
+ * {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader, Collection, float)}
+ */
+ @Deprecated
+ public int addStopWords(IndexReader reader, String fieldName, float maxPercentDocs) throws IOException {
+ return addStopWords(reader, fieldName, (int) (reader.numDocs() * maxPercentDocs));
+ }
+
+ /**
+ * Automatically adds stop words for the given field with terms exceeding the maxPercentDocs
+ *
+ * @param reader The {@link IndexReader} which will be consulted to identify potential stop words that
+ * exceed the required document frequency
+ * @param fieldName The field for which stopwords will be added
+ * @param maxDocFreq The maximum number of index documents which
+ * can contain a term, after which the term is considered to be a stop word.
+ * @return The number of stop words identified.
+ * @throws IOException
+ * @deprecated Stowords should be calculated at instantiation using
+ * {@link #QueryAutoStopWordAnalyzer(Version, Analyzer, IndexReader, Collection, int)}
+ */
+ @Deprecated
+ public int addStopWords(IndexReader reader, String fieldName, int maxDocFreq) throws IOException {
+ HashSet<String> stopWords = new HashSet<String>();
+ String internedFieldName = StringHelper.intern(fieldName);
+ TermEnum te = reader.terms(new Term(fieldName));
+ Term term = te.term();
+ while (term != null) {
+ if (term.field() != internedFieldName) {
+ break;
+ }
+ if (te.docFreq() > maxDocFreq) {
+ stopWords.add(term.text());
+ }
+ if (!te.next()) {
+ break;
+ }
+ term = te.term();
+ }
+ stopWordsPerField.put(fieldName, stopWords);
+
+ /* if the stopwords for a field are changed,
+ * then saved streams for that field are erased.
+ */
+ @SuppressWarnings("unchecked")
+ Map<String,SavedStreams> streamMap = (Map<String,SavedStreams>) getPreviousTokenStream();
+ if (streamMap != null)
+ streamMap.remove(fieldName);
+
+ return stopWords.size();
+ }
+
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ TokenStream result;
+ try {
+ result = delegate.reusableTokenStream(fieldName, reader);
+ } catch (IOException e) {
+ result = delegate.tokenStream(fieldName, reader);
+ }
+ Set<String> stopWords = stopWordsPerField.get(fieldName);
+ if (stopWords != null) {
+ result = new StopFilter(matchVersion, result, stopWords);
+ }
+ return result;
+ }
+
+ private class SavedStreams {
+ /* the underlying stream */
+ TokenStream wrapped;
+
+ /*
+ * when there are no stopwords for the field, refers to wrapped.
+ * if there stopwords, it is a StopFilter around wrapped.
+ */
+ TokenStream withStopFilter;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public TokenStream reusableTokenStream(String fieldName, Reader reader)
+ throws IOException {
+ /* map of SavedStreams for each field */
+ Map<String,SavedStreams> streamMap = (Map<String,SavedStreams>) getPreviousTokenStream();
+ if (streamMap == null) {
+ streamMap = new HashMap<String, SavedStreams>();
+ setPreviousTokenStream(streamMap);
+ }
+
+ SavedStreams streams = streamMap.get(fieldName);
+ if (streams == null) {
+ /* an entry for this field does not exist, create one */
+ streams = new SavedStreams();
+ streamMap.put(fieldName, streams);
+ streams.wrapped = delegate.reusableTokenStream(fieldName, reader);
+
+ /* if there are any stopwords for the field, save the stopfilter */
+ Set<String> stopWords = stopWordsPerField.get(fieldName);
+ if (stopWords != null) {
+ streams.withStopFilter = new StopFilter(matchVersion, streams.wrapped, stopWords);
+ } else {
+ streams.withStopFilter = streams.wrapped;
+ }
+
+ } else {
+ /*
+ * an entry for this field exists, verify the wrapped stream has not
+ * changed. if it has not, reuse it, otherwise wrap the new stream.
+ */
+ TokenStream result = delegate.reusableTokenStream(fieldName, reader);
+ if (result == streams.wrapped) {
+ /* the wrapped analyzer reused the stream */
+ } else {
+ /*
+ * the wrapped analyzer did not. if there are any stopwords for the
+ * field, create a new StopFilter around the new stream
+ */
+ streams.wrapped = result;
+ Set<String> stopWords = stopWordsPerField.get(fieldName);
+ if (stopWords != null) {
+ streams.withStopFilter = new StopFilter(matchVersion, streams.wrapped, stopWords);
+ } else {
+ streams.withStopFilter = streams.wrapped;
+ }
+ }
+ }
+
+ return streams.withStopFilter;
+ }
+
+ /**
+ * Provides information on which stop words have been identified for a field
+ *
+ * @param fieldName The field for which stop words identified in "addStopWords"
+ * method calls will be returned
+ * @return the stop words identified for a field
+ */
+ public String[] getStopWords(String fieldName) {
+ Set<String> stopWords = stopWordsPerField.get(fieldName);
+ return stopWords != null ? stopWords.toArray(new String[stopWords.size()]) : new String[0];
+ }
+
+ /**
+ * Provides information on which stop words have been identified for all fields
+ *
+ * @return the stop words (as terms)
+ */
+ public Term[] getStopWords() {
+ List<Term> allStopWords = new ArrayList<Term>();
+ for (String fieldName : stopWordsPerField.keySet()) {
+ Set<String> stopWords = stopWordsPerField.get(fieldName);
+ for (String text : stopWords) {
+ allStopWords.add(new Term(fieldName, text));
+ }
+ }
+ return allStopWords.toArray(new Term[allStopWords.size()]);
+ }
+
+}
fnp / pylucene.git / blobdiff