+++ /dev/null
-/**
- * Copyright 2004-2005 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.search.similar;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.PrintStream;
-import java.io.Reader;
-import java.io.StringReader;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermFreqVector;
-import org.apache.lucene.search.BooleanClause;
-import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.DefaultSimilarity;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Query;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.TopDocs;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.PriorityQueue;
-import org.apache.lucene.util.Version;
-
-/**
- * Generate "more like this" similarity queries. Based on this mail: <code><pre>
- * Lucene does let you access the document frequency of terms, with IndexReader.docFreq().
- * Term frequencies can be computed by re-tokenizing the text, which, for a single document,
- * is usually fast enough. But looking up the docFreq() of every term in the document is
- * probably too slow.
- *
- * You can use some heuristics to prune the set of terms, to avoid calling docFreq() too much,
- * or at all. Since you're trying to maximize a tf*idf score, you're probably most interested
- * in terms with a high tf. Choosing a tf threshold even as low as two or three will radically
- * reduce the number of terms under consideration. Another heuristic is that terms with a
- * high idf (i.e., a low df) tend to be longer. So you could threshold the terms by the
- * number of characters, not selecting anything less than, e.g., six or seven characters.
- * With these sorts of heuristics you can usually find small set of, e.g., ten or fewer terms
- * that do a pretty good job of characterizing a document.
- *
- * It all depends on what you're trying to do. If you're trying to eek out that last percent
- * of precision and recall regardless of computational difficulty so that you can win a TREC
- * competition, then the techniques I mention above are useless. But if you're trying to
- * provide a "more like this" button on a search results page that does a decent job and has
- * good performance, such techniques might be useful.
- *
- * An efficient, effective "more-like-this" query generator would be a great contribution, if
- * anyone's interested. I'd imagine that it would take a Reader or a String (the document's
- * text), analyzer Analyzer, and return a set of representative terms using heuristics like those
- * above. The frequency and length thresholds could be parameters, etc.
- *
- * Doug
- * </pre></code>
- *
- *
- * <p>
- * <h3>Initial Usage</h3>
- * <p/>
- * This class has lots of options to try to make it efficient and flexible.
- * The simplest possible usage is as follows. The bold
- * fragment is specific to this class.
- *
- * <pre class="prettyprint">
- *
- * IndexReader ir = ...
- * IndexSearcher is = ...
- *
- * MoreLikeThis mlt = new MoreLikeThis(ir);
- * Reader target = ... // orig source of doc you want to find similarities to
- * Query query = mlt.like( target);
- *
- * Hits hits = is.search(query);
- * // now the usual iteration thru 'hits' - the only thing to watch for is to make sure
- * //you ignore the doc if it matches your 'target' document, as it should be similar to itself
- *
- * </pre>
- *
- * Thus you:
- * <ol>
- * <li>do your normal, Lucene setup for searching,
- * <li>create a MoreLikeThis,
- * <li>get the text of the doc you want to find similarities to
- * <li>then call one of the like() calls to generate a similarity query
- * <li>call the searcher to find the similar docs
- * </ol>
- *
- * <h3>More Advanced Usage</h3>
- *
- * You may want to use {@link #setFieldNames setFieldNames(...)} so you can
- * examine multiple fields (e.g. body and title) for similarity.
- * <p>
- *
- * Depending on the size of your index and the size and makeup of your documents
- * you may want to call the other set methods to control how the similarity
- * queries are generated:
- * <ul>
- * <li> {@link #setMinTermFreq setMinTermFreq(...)}
- * <li> {@link #setMinDocFreq setMinDocFreq(...)}
- * <li> {@link #setMaxDocFreq setMaxDocFreq(...)}
- * <li> {@link #setMaxDocFreqPct setMaxDocFreqPct(...)}
- * <li> {@link #setMinWordLen setMinWordLen(...)}
- * <li> {@link #setMaxWordLen setMaxWordLen(...)}
- * <li> {@link #setMaxQueryTerms setMaxQueryTerms(...)}
- * <li> {@link #setMaxNumTokensParsed setMaxNumTokensParsed(...)}
- * <li> {@link #setStopWords setStopWord(...)}
- * </ul>
- *
- * <hr>
- *
- * <pre>
- * Changes: Mark Harwood 29/02/04
- * Some bugfixing, some refactoring, some optimisation.
- * - bugfix: retrieveTerms(int docNum) was not working for indexes without a termvector -added missing code
- * - bugfix: No significant terms being created for fields with a termvector - because
- * was only counting one occurrence per term/field pair in calculations(ie not including frequency info from TermVector)
- * - refactor: moved common code into isNoiseWord()
- * - optimise: when no termvector support available - used maxNumTermsParsed to limit amount of tokenization
- * </pre>
- *
- */
-public final class MoreLikeThis {
-
- /**
- * Default maximum number of tokens to parse in each example doc field that is
- * not stored with TermVector support.
- *
- * @see #getMaxNumTokensParsed
- */
- public static final int DEFAULT_MAX_NUM_TOKENS_PARSED = 5000;
-
- /**
- * Default analyzer to parse source doc with.
- *
- * @see #getAnalyzer
- * @deprecated This default will be removed in Lucene 4.0 (with the default
- * being null). If you are not using term vectors, explicitly set
- * your analyzer instead.
- */
- @Deprecated
- public static final Analyzer DEFAULT_ANALYZER = new StandardAnalyzer(
- Version.LUCENE_CURRENT);
-
- /**
- * Ignore terms with less than this frequency in the source doc.
- *
- * @see #getMinTermFreq
- * @see #setMinTermFreq
- */
- public static final int DEFAULT_MIN_TERM_FREQ = 2;
-
- /**
- * Ignore words which do not occur in at least this many docs.
- *
- * @see #getMinDocFreq
- * @see #setMinDocFreq
- */
- public static final int DEFAULT_MIN_DOC_FREQ = 5;
-
- /**
- * Ignore words which occur in more than this many docs.
- *
- * @see #getMaxDocFreq
- * @see #setMaxDocFreq
- * @see #setMaxDocFreqPct
- */
- public static final int DEFAULT_MAX_DOC_FREQ = Integer.MAX_VALUE;
-
- /**
- * Boost terms in query based on score.
- *
- * @see #isBoost
- * @see #setBoost
- */
- public static final boolean DEFAULT_BOOST = false;
-
- /**
- * Default field names. Null is used to specify that the field names should be
- * looked up at runtime from the provided reader.
- */
- public static final String[] DEFAULT_FIELD_NAMES = new String[] {"contents"};
-
- /**
- * Ignore words less than this length or if 0 then this has no effect.
- *
- * @see #getMinWordLen
- * @see #setMinWordLen
- */
- public static final int DEFAULT_MIN_WORD_LENGTH = 0;
-
- /**
- * Ignore words greater than this length or if 0 then this has no effect.
- *
- * @see #getMaxWordLen
- * @see #setMaxWordLen
- */
- public static final int DEFAULT_MAX_WORD_LENGTH = 0;
-
- /**
- * Default set of stopwords. If null means to allow stop words.
- *
- * @see #setStopWords
- * @see #getStopWords
- */
- public static final Set<?> DEFAULT_STOP_WORDS = null;
-
- /**
- * Current set of stop words.
- */
- private Set<?> stopWords = DEFAULT_STOP_WORDS;
-
- /**
- * Return a Query with no more than this many terms.
- *
- * @see BooleanQuery#getMaxClauseCount
- * @see #getMaxQueryTerms
- * @see #setMaxQueryTerms
- */
- public static final int DEFAULT_MAX_QUERY_TERMS = 25;
-
- /**
- * Analyzer that will be used to parse the doc.
- */
- private Analyzer analyzer = DEFAULT_ANALYZER;
-
- /**
- * Ignore words less frequent that this.
- */
- private int minTermFreq = DEFAULT_MIN_TERM_FREQ;
-
- /**
- * Ignore words which do not occur in at least this many docs.
- */
- private int minDocFreq = DEFAULT_MIN_DOC_FREQ;
-
- /**
- * Ignore words which occur in more than this many docs.
- */
- private int maxDocFreq = DEFAULT_MAX_DOC_FREQ;
-
- /**
- * Should we apply a boost to the Query based on the scores?
- */
- private boolean boost = DEFAULT_BOOST;
-
- /**
- * Field name we'll analyze.
- */
- private String[] fieldNames = DEFAULT_FIELD_NAMES;
-
- /**
- * The maximum number of tokens to parse in each example doc field that is not
- * stored with TermVector support
- */
- private int maxNumTokensParsed = DEFAULT_MAX_NUM_TOKENS_PARSED;
-
- /**
- * Ignore words if less than this len.
- */
- private int minWordLen = DEFAULT_MIN_WORD_LENGTH;
-
- /**
- * Ignore words if greater than this len.
- */
- private int maxWordLen = DEFAULT_MAX_WORD_LENGTH;
-
- /**
- * Don't return a query longer than this.
- */
- private int maxQueryTerms = DEFAULT_MAX_QUERY_TERMS;
-
- /**
- * For idf() calculations.
- */
- private Similarity similarity;// = new DefaultSimilarity();
-
- /**
- * IndexReader to use
- */
- private final IndexReader ir;
-
- /**
- * Boost factor to use when boosting the terms
- */
- private float boostFactor = 1;
-
- /**
- * Returns the boost factor used when boosting terms
- *
- * @return the boost factor used when boosting terms
- */
- public float getBoostFactor() {
- return boostFactor;
- }
-
- /**
- * Sets the boost factor to use when boosting terms
- *
- * @param boostFactor
- */
- public void setBoostFactor(float boostFactor) {
- this.boostFactor = boostFactor;
- }
-
- /**
- * Constructor requiring an IndexReader.
- */
- public MoreLikeThis(IndexReader ir) {
- this(ir, new DefaultSimilarity());
- }
-
- public MoreLikeThis(IndexReader ir, Similarity sim) {
- this.ir = ir;
- this.similarity = sim;
- }
-
- public Similarity getSimilarity() {
- return similarity;
- }
-
- public void setSimilarity(Similarity similarity) {
- this.similarity = similarity;
- }
-
- /**
- * Returns an analyzer that will be used to parse source doc with. The default
- * analyzer is the {@link #DEFAULT_ANALYZER}.
- *
- * @return the analyzer that will be used to parse source doc with.
- * @see #DEFAULT_ANALYZER
- */
- public Analyzer getAnalyzer() {
- return analyzer;
- }
-
- /**
- * Sets the analyzer to use. An analyzer is not required for generating a
- * query with the {@link #like(int)} method, all other 'like' methods require
- * an analyzer.
- *
- * @param analyzer
- * the analyzer to use to tokenize text.
- */
- public void setAnalyzer(Analyzer analyzer) {
- this.analyzer = analyzer;
- }
-
- /**
- * Returns the frequency below which terms will be ignored in the source doc.
- * The default frequency is the {@link #DEFAULT_MIN_TERM_FREQ}.
- *
- * @return the frequency below which terms will be ignored in the source doc.
- */
- public int getMinTermFreq() {
- return minTermFreq;
- }
-
- /**
- * Sets the frequency below which terms will be ignored in the source doc.
- *
- * @param minTermFreq
- * the frequency below which terms will be ignored in the source doc.
- */
- public void setMinTermFreq(int minTermFreq) {
- this.minTermFreq = minTermFreq;
- }
-
- /**
- * Returns the frequency at which words will be ignored which do not occur in
- * at least this many docs. The default frequency is
- * {@link #DEFAULT_MIN_DOC_FREQ}.
- *
- * @return the frequency at which words will be ignored which do not occur in
- * at least this many docs.
- */
- public int getMinDocFreq() {
- return minDocFreq;
- }
-
- /**
- * Sets the frequency at which words will be ignored which do not occur in at
- * least this many docs.
- *
- * @param minDocFreq
- * the frequency at which words will be ignored which do not occur in
- * at least this many docs.
- */
- public void setMinDocFreq(int minDocFreq) {
- this.minDocFreq = minDocFreq;
- }
-
- /**
- * Returns the maximum frequency in which words may still appear. Words that
- * appear in more than this many docs will be ignored. The default frequency
- * is {@link #DEFAULT_MAX_DOC_FREQ}.
- *
- * @return get the maximum frequency at which words are still allowed, words
- * which occur in more docs than this are ignored.
- */
- public int getMaxDocFreq() {
- return maxDocFreq;
- }
-
- /**
- * Set the maximum frequency in which words may still appear. Words that
- * appear in more than this many docs will be ignored.
- *
- * @param maxFreq
- * the maximum count of documents that a term may appear in to be
- * still considered relevant
- */
- public void setMaxDocFreq(int maxFreq) {
- this.maxDocFreq = maxFreq;
- }
-
- /**
- * Set the maximum percentage in which words may still appear. Words that
- * appear in more than this many percent of all docs will be ignored.
- *
- * @param maxPercentage
- * the maximum percentage of documents (0-100) that a term may appear
- * in to be still considered relevant
- */
- public void setMaxDocFreqPct(int maxPercentage) {
- this.maxDocFreq = maxPercentage * ir.numDocs() / 100;
- }
-
- /**
- * Returns whether to boost terms in query based on "score" or not. The
- * default is {@link #DEFAULT_BOOST}.
- *
- * @return whether to boost terms in query based on "score" or not.
- * @see #setBoost
- */
- public boolean isBoost() {
- return boost;
- }
-
- /**
- * Sets whether to boost terms in query based on "score" or not.
- *
- * @param boost
- * true to boost terms in query based on "score", false otherwise.
- * @see #isBoost
- */
- public void setBoost(boolean boost) {
- this.boost = boost;
- }
-
- /**
- * Returns the field names that will be used when generating the 'More Like
- * This' query. The default field names that will be used is
- * {@link #DEFAULT_FIELD_NAMES}.
- *
- * @return the field names that will be used when generating the 'More Like
- * This' query.
- */
- public String[] getFieldNames() {
- return fieldNames;
- }
-
- /**
- * Sets the field names that will be used when generating the 'More Like This'
- * query. Set this to null for the field names to be determined at runtime
- * from the IndexReader provided in the constructor.
- *
- * @param fieldNames
- * the field names that will be used when generating the 'More Like
- * This' query.
- */
- public void setFieldNames(String[] fieldNames) {
- this.fieldNames = fieldNames;
- }
-
- /**
- * Returns the minimum word length below which words will be ignored. Set this
- * to 0 for no minimum word length. The default is
- * {@link #DEFAULT_MIN_WORD_LENGTH}.
- *
- * @return the minimum word length below which words will be ignored.
- */
- public int getMinWordLen() {
- return minWordLen;
- }
-
- /**
- * Sets the minimum word length below which words will be ignored.
- *
- * @param minWordLen
- * the minimum word length below which words will be ignored.
- */
- public void setMinWordLen(int minWordLen) {
- this.minWordLen = minWordLen;
- }
-
- /**
- * Returns the maximum word length above which words will be ignored. Set this
- * to 0 for no maximum word length. The default is
- * {@link #DEFAULT_MAX_WORD_LENGTH}.
- *
- * @return the maximum word length above which words will be ignored.
- */
- public int getMaxWordLen() {
- return maxWordLen;
- }
-
- /**
- * Sets the maximum word length above which words will be ignored.
- *
- * @param maxWordLen
- * the maximum word length above which words will be ignored.
- */
- public void setMaxWordLen(int maxWordLen) {
- this.maxWordLen = maxWordLen;
- }
-
- /**
- * Set the set of stopwords. Any word in this set is considered
- * "uninteresting" and ignored. Even if your Analyzer allows stopwords, you
- * might want to tell the MoreLikeThis code to ignore them, as for the
- * purposes of document similarity it seems reasonable to assume that
- * "a stop word is never interesting".
- *
- * @param stopWords
- * set of stopwords, if null it means to allow stop words
- *
- * @see org.apache.lucene.analysis.StopFilter#makeStopSet
- * StopFilter.makeStopSet()
- * @see #getStopWords
- */
- public void setStopWords(Set<?> stopWords) {
- this.stopWords = stopWords;
- }
-
- /**
- * Get the current stop words being used.
- *
- * @see #setStopWords
- */
- public Set<?> getStopWords() {
- return stopWords;
- }
-
- /**
- * Returns the maximum number of query terms that will be included in any
- * generated query. The default is {@link #DEFAULT_MAX_QUERY_TERMS}.
- *
- * @return the maximum number of query terms that will be included in any
- * generated query.
- */
- public int getMaxQueryTerms() {
- return maxQueryTerms;
- }
-
- /**
- * Sets the maximum number of query terms that will be included in any
- * generated query.
- *
- * @param maxQueryTerms
- * the maximum number of query terms that will be included in any
- * generated query.
- */
- public void setMaxQueryTerms(int maxQueryTerms) {
- this.maxQueryTerms = maxQueryTerms;
- }
-
- /**
- * @return The maximum number of tokens to parse in each example doc field
- * that is not stored with TermVector support
- * @see #DEFAULT_MAX_NUM_TOKENS_PARSED
- */
- public int getMaxNumTokensParsed() {
- return maxNumTokensParsed;
- }
-
- /**
- * @param i
- * The maximum number of tokens to parse in each example doc field
- * that is not stored with TermVector support
- */
- public void setMaxNumTokensParsed(int i) {
- maxNumTokensParsed = i;
- }
-
- /**
- * Return a query that will return docs like the passed lucene document ID.
- *
- * @param docNum
- * the documentID of the lucene doc to generate the 'More Like This"
- * query for.
- * @return a query that will return docs like the passed lucene document ID.
- */
- public Query like(int docNum) throws IOException {
- if (fieldNames == null) {
- // gather list of valid fields from lucene
- Collection<String> fields = ir
- .getFieldNames(IndexReader.FieldOption.INDEXED);
- fieldNames = fields.toArray(new String[fields.size()]);
- }
-
- return createQuery(retrieveTerms(docNum));
- }
-
- /**
- * Return a query that will return docs like the passed file.
- *
- * @return a query that will return docs like the passed file.
- * @deprecated use {@link #like(Reader, String)} instead */
- @Deprecated
- public Query like(File f) throws IOException {
- if (fieldNames == null) {
- // gather list of valid fields from lucene
- Collection<String> fields = ir
- .getFieldNames(IndexReader.FieldOption.INDEXED);
- fieldNames = fields.toArray(new String[fields.size()]);
- }
-
- return like(new FileReader(f));
- }
-
- /**
- * Return a query that will return docs like the passed URL.
- *
- * @return a query that will return docs like the passed URL.
- * @deprecated use {@link #like(Reader, String)} instead */
- @Deprecated
- public Query like(URL u) throws IOException {
- return like(new InputStreamReader(u.openConnection().getInputStream()));
- }
-
- /**
- * Return a query that will return docs like the passed stream.
- *
- * @return a query that will return docs like the passed stream.
- * @deprecated use {@link #like(Reader, String)} instead */
- @Deprecated
- public Query like(java.io.InputStream is) throws IOException {
- return like(new InputStreamReader(is));
- }
-
- /** @deprecated use {@link #like(Reader, String)} instead */
- @Deprecated
- public Query like(Reader r) throws IOException {
- return createQuery(retrieveTerms(r, fieldNames[0]));
- }
-
- /**
- * Return a query that will return docs like the passed Reader.
- *
- * @return a query that will return docs like the passed Reader.
- */
- public Query like(Reader r, String fieldName) throws IOException {
- return createQuery(retrieveTerms(r, fieldName));
- }
-
- /**
- * Create the More like query from a PriorityQueue
- */
- private Query createQuery(PriorityQueue<Object[]> q) {
- BooleanQuery query = new BooleanQuery();
- Object cur;
- int qterms = 0;
- float bestScore = 0;
-
- while (((cur = q.pop()) != null)) {
- Object[] ar = (Object[]) cur;
- TermQuery tq = new TermQuery(new Term((String) ar[1], (String) ar[0]));
-
- if (boost) {
- if (qterms == 0) {
- bestScore = ((Float) ar[2]).floatValue();
- }
- float myScore = ((Float) ar[2]).floatValue();
-
- tq.setBoost(boostFactor * myScore / bestScore);
- }
-
- try {
- query.add(tq, BooleanClause.Occur.SHOULD);
- } catch (BooleanQuery.TooManyClauses ignore) {
- break;
- }
-
- qterms++;
- if (maxQueryTerms > 0 && qterms >= maxQueryTerms) {
- break;
- }
- }
-
- return query;
- }
-
- /**
- * Create a PriorityQueue from a word->tf map.
- *
- * @param words
- * a map of words keyed on the word(String) with Int objects as the
- * values.
- */
- private PriorityQueue<Object[]> createQueue(Map<String,Int> words)
- throws IOException {
- // have collected all words in doc and their freqs
- int numDocs = ir.numDocs();
- FreqQ res = new FreqQ(words.size()); // will order words by score
-
- Iterator<String> it = words.keySet().iterator();
- while (it.hasNext()) { // for every word
- String word = it.next();
-
- int tf = words.get(word).x; // term freq in the source doc
- if (minTermFreq > 0 && tf < minTermFreq) {
- continue; // filter out words that don't occur enough times in the
- // source
- }
-
- // go through all the fields and find the largest document frequency
- String topField = fieldNames[0];
- int docFreq = 0;
- for (int i = 0; i < fieldNames.length; i++) {
- int freq = ir.docFreq(new Term(fieldNames[i], word));
- topField = (freq > docFreq) ? fieldNames[i] : topField;
- docFreq = (freq > docFreq) ? freq : docFreq;
- }
-
- if (minDocFreq > 0 && docFreq < minDocFreq) {
- continue; // filter out words that don't occur in enough docs
- }
-
- if (docFreq > maxDocFreq) {
- continue; // filter out words that occur in too many docs
- }
-
- if (docFreq == 0) {
- continue; // index update problem?
- }
-
- float idf = similarity.idf(docFreq, numDocs);
- float score = tf * idf;
-
- // only really need 1st 3 entries, other ones are for troubleshooting
- res.insertWithOverflow(new Object[] {word, // the word
- topField, // the top field
- Float.valueOf(score), // overall score
- Float.valueOf(idf), // idf
- Integer.valueOf(docFreq), // freq in all docs
- Integer.valueOf(tf)});
- }
- return res;
- }
-
- /**
- * Describe the parameters that control how the "more like this" query is
- * formed.
- */
- public String describeParams() {
- StringBuilder sb = new StringBuilder();
- sb.append("\t" + "maxQueryTerms : " + maxQueryTerms + "\n");
- sb.append("\t" + "minWordLen : " + minWordLen + "\n");
- sb.append("\t" + "maxWordLen : " + maxWordLen + "\n");
- sb.append("\t" + "fieldNames : ");
- String delim = "";
- for (int i = 0; i < fieldNames.length; i++) {
- String fieldName = fieldNames[i];
- sb.append(delim).append(fieldName);
- delim = ", ";
- }
- sb.append("\n");
- sb.append("\t" + "boost : " + boost + "\n");
- sb.append("\t" + "minTermFreq : " + minTermFreq + "\n");
- sb.append("\t" + "minDocFreq : " + minDocFreq + "\n");
- return sb.toString();
- }
-
- /**
- * Find words for a more-like-this query former.
- *
- * @param docNum
- * the id of the lucene document from which to find terms
- */
- public PriorityQueue<Object[]> retrieveTerms(int docNum) throws IOException {
- Map<String,Int> termFreqMap = new HashMap<String,Int>();
- for (int i = 0; i < fieldNames.length; i++) {
- String fieldName = fieldNames[i];
- TermFreqVector vector = ir.getTermFreqVector(docNum, fieldName);
-
- // field does not store term vector info
- if (vector == null) {
- Document d = ir.document(docNum);
- String text[] = d.getValues(fieldName);
- if (text != null) {
- for (int j = 0; j < text.length; j++) {
- addTermFrequencies(new StringReader(text[j]), termFreqMap,
- fieldName);
- }
- }
- } else {
- addTermFrequencies(termFreqMap, vector);
- }
-
- }
-
- return createQueue(termFreqMap);
- }
-
- /**
- * Adds terms and frequencies found in vector into the Map termFreqMap
- *
- * @param termFreqMap
- * a Map of terms and their frequencies
- * @param vector
- * List of terms and their frequencies for a doc/field
- */
- private void addTermFrequencies(Map<String,Int> termFreqMap,
- TermFreqVector vector) {
- String[] terms = vector.getTerms();
- int freqs[] = vector.getTermFrequencies();
- for (int j = 0; j < terms.length; j++) {
- String term = terms[j];
-
- if (isNoiseWord(term)) {
- continue;
- }
- // increment frequency
- Int cnt = termFreqMap.get(term);
- if (cnt == null) {
- cnt = new Int();
- termFreqMap.put(term, cnt);
- cnt.x = freqs[j];
- } else {
- cnt.x += freqs[j];
- }
- }
- }
-
- /**
- * Adds term frequencies found by tokenizing text from reader into the Map
- * words
- *
- * @param r
- * a source of text to be tokenized
- * @param termFreqMap
- * a Map of terms and their frequencies
- * @param fieldName
- * Used by analyzer for any special per-field analysis
- */
- private void addTermFrequencies(Reader r, Map<String,Int> termFreqMap,
- String fieldName) throws IOException {
- TokenStream ts = analyzer.reusableTokenStream(fieldName, r);
- int tokenCount = 0;
- // for every token
- CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
- ts.reset();
- while (ts.incrementToken()) {
- String word = termAtt.toString();
- tokenCount++;
- if (tokenCount > maxNumTokensParsed) {
- break;
- }
- if (isNoiseWord(word)) {
- continue;
- }
-
- // increment frequency
- Int cnt = termFreqMap.get(word);
- if (cnt == null) {
- termFreqMap.put(word, new Int());
- } else {
- cnt.x++;
- }
- }
- ts.end();
- ts.close();
- }
-
- /**
- * determines if the passed term is likely to be of interest in "more like"
- * comparisons
- *
- * @param term
- * The word being considered
- * @return true if should be ignored, false if should be used in further
- * analysis
- */
- private boolean isNoiseWord(String term) {
- int len = term.length();
- if (minWordLen > 0 && len < minWordLen) {
- return true;
- }
- if (maxWordLen > 0 && len > maxWordLen) {
- return true;
- }
- if (stopWords != null && stopWords.contains(term)) {
- return true;
- }
- return false;
- }
-
- /**
- * Find words for a more-like-this query former. The result is a priority
- * queue of arrays with one entry for <b>every word</b> in the document. Each
- * array has 6 elements. The elements are:
- * <ol>
- * <li>The word (String)
- * <li>The top field that this word comes from (String)
- * <li>The score for this word (Float)
- * <li>The IDF value (Float)
- * <li>The frequency of this word in the index (Integer)
- * <li>The frequency of this word in the source document (Integer)
- * </ol>
- * This is a somewhat "advanced" routine, and in general only the 1st entry in the array is of interest.
- * This method is exposed so that you can identify the "interesting words" in a document.
- * For an easier method to call see {@link #retrieveInterestingTerms retrieveInterestingTerms()}.
- *
- * @param r the reader that has the content of the document
- * @param fieldName field passed to the analyzer to use when analyzing the content
- * @return the most interesting words in the document ordered by score, with the highest scoring, or best entry, first
- * @see #retrieveInterestingTerms
- */
- public PriorityQueue<Object[]> retrieveTerms(Reader r, String fieldName) throws IOException {
- Map<String, Int> words = new HashMap<String, Int>();
- addTermFrequencies(r, words, fieldName);
- return createQueue(words);
- }
-
- /** @deprecated use {@link #retrieveTerms(Reader, String)} instead */
- @Deprecated
- public PriorityQueue<Object[]> retrieveTerms(Reader r) throws IOException {
- return retrieveTerms(r, fieldNames[0]);
- }
-
- /**
- * @see #retrieveInterestingTerms(java.io.Reader, String)
- */
- public String[] retrieveInterestingTerms(int docNum) throws IOException {
- ArrayList<Object> al = new ArrayList<Object>(maxQueryTerms);
- PriorityQueue<Object[]> pq = retrieveTerms(docNum);
- Object cur;
- int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all
- // words but that's probably not useful to our
- // caller...
- // we just want to return the top words
- while (((cur = pq.pop()) != null) && lim-- > 0) {
- Object[] ar = (Object[]) cur;
- al.add(ar[0]); // the 1st entry is the interesting word
- }
- String[] res = new String[al.size()];
- return al.toArray(res);
- }
-
- /**
- * Convenience routine to make it easy to return the most interesting words in a document.
- * More advanced users will call {@link #retrieveTerms(Reader, String) retrieveTerms()} directly.
- *
- * @param r the source document
- * @param fieldName field passed to analyzer to use when analyzing the content
- * @return the most interesting words in the document
- * @see #retrieveTerms(java.io.Reader, String)
- * @see #setMaxQueryTerms
- */
- public String[] retrieveInterestingTerms(Reader r, String fieldName) throws IOException {
- ArrayList<Object> al = new ArrayList<Object>(maxQueryTerms);
- PriorityQueue<Object[]> pq = retrieveTerms(r, fieldName);
- Object cur;
- int lim = maxQueryTerms; // have to be careful, retrieveTerms returns all
- // words but that's probably not useful to our
- // caller...
- // we just want to return the top words
- while (((cur = pq.pop()) != null) && lim-- > 0) {
- Object[] ar = (Object[]) cur;
- al.add(ar[0]); // the 1st entry is the interesting word
- }
- String[] res = new String[al.size()];
- return al.toArray(res);
- }
-
- /** @deprecated use {@link #retrieveInterestingTerms(Reader, String)} instead. */
- @Deprecated
- public String[] retrieveInterestingTerms(Reader r) throws IOException {
- return retrieveInterestingTerms(r, fieldNames[0]);
- }
-
- /**
- * PriorityQueue that orders words by score.
- */
- private static class FreqQ extends PriorityQueue<Object[]> {
- FreqQ(int s) {
- initialize(s);
- }
-
- @Override
- protected boolean lessThan(Object[] aa, Object[] bb) {
- Float fa = (Float) aa[2];
- Float fb = (Float) bb[2];
- return fa.floatValue() > fb.floatValue();
- }
- }
-
- /**
- * Use for frequencies and to avoid renewing Integers.
- */
- private static class Int {
- int x;
-
- Int() {
- x = 1;
- }
- }
-
-}
fnp / pylucene.git / blobdiff