+++ /dev/null
-package org.apache.lucene.misc;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
-import org.apache.lucene.index.TermEnum;
-import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.util.PriorityQueue;
-import java.util.Arrays;
-import java.util.Comparator;
-
-import java.io.File;
-
-/**
- * <code>HighFreqTerms</code> class extracts the top n most frequent terms
- * (by document frequency ) from an existing Lucene index and reports their
- * document frequency. If used with the -t flag it also reports their
- * total tf (total number of occurences) in order of highest total tf
- */
-public class HighFreqTerms {
-
- // The top numTerms will be displayed
- public static final int DEFAULTnumTerms = 100;
- public static int numTerms = DEFAULTnumTerms;
-
- public static void main(String[] args) throws Exception {
- IndexReader reader = null;
- FSDirectory dir = null;
- String field = null;
- boolean IncludeTermFreqs = false;
-
- if (args.length == 0 || args.length > 4) {
- usage();
- System.exit(1);
- }
-
- if (args.length > 0) {
- dir = FSDirectory.open(new File(args[0]));
- }
-
- for (int i = 1; i < args.length; i++) {
- if (args[i].equals("-t")) {
- IncludeTermFreqs = true;
- }
- else{
- try {
- numTerms = Integer.parseInt(args[i]);
- } catch (NumberFormatException e) {
- field=args[i];
- }
- }
- }
-
-
- reader = IndexReader.open(dir, true);
- TermStats[] terms = getHighFreqTerms(reader, numTerms, field);
- /*
- * Insert logic so it will only lookup totaltf if right arg
- * also change names as in flex
- */
- if (!IncludeTermFreqs) {
- //default HighFreqTerms behavior
- for (int i = 0; i < terms.length; i++) {
- System.out.printf("%s %,d \n",
- terms[i].term, terms[i].docFreq);
- }
- } else {
-
- TermStats[] termsWithTF = sortByTotalTermFreq(reader, terms);
- for (int i = 0; i < termsWithTF.length; i++) {
- System.out.printf("%s \t total_tf = %,d \t doc freq = %,d \n",
- termsWithTF[i].term, termsWithTF[i].totalTermFreq, termsWithTF[i].docFreq);
- }
- }
-
- reader.close();
- }
-
- private static void usage() {
- System.out
- .println("\n\n"
- + "java org.apache.lucene.misc.HighFreqTerms <index dir> [-t] [number_terms] [field]\n\t -t: include totalTermFreq\n\n");
- }
-
- /**
- *
- * @param reader
- * @param numTerms
- * @param field
- * @return TermStats[] ordered by terms with highest docFreq first.
- * @throws Exception
- */
- public static TermStats[] getHighFreqTerms(IndexReader reader,
- int numTerms, String field) throws Exception {
-
- TermInfoWiTFQueue tiq = new TermInfoWiTFQueue(numTerms);
- if (field != null) {
- TermEnum terms = reader.terms(new Term(field));
- if (terms != null && terms.term() != null) {
- do {
- if (!terms.term().field().equals(field)) {
- break;
- }
- tiq.insertWithOverflow(new TermStats(terms.term(), terms.docFreq()));
- } while (terms.next());
- } else {
- System.out.println("No terms for field \"" + field + "\"");
- }
- } else {
- TermEnum terms = reader.terms();
- while (terms.next()) {
- tiq.insertWithOverflow(new TermStats(terms.term(), terms.docFreq()));
- }
- }
-
- TermStats[] result = new TermStats[tiq.size()];
-
- // we want highest first so we read the queue and populate the array
- // starting at the end and work backwards
- int count = tiq.size() - 1;
- while (tiq.size() != 0) {
- result[count] = tiq.pop();
- count--;
- }
- return result;
- }
-
- /**
- * Takes array of TermStats. For each term looks up the tf for each doc
- * containing the term and stores the total in the output array of TermStats.
- * Output array is sorted by highest total tf.
- *
- * @param reader
- * @param terms
- * TermStats[]
- * @return TermStats[]
- * @throws Exception
- */
-
- public static TermStats[] sortByTotalTermFreq(IndexReader reader, TermStats[] terms) throws Exception {
- TermStats[] ts = new TermStats[terms.length]; // array for sorting
- long totalTF;
- for (int i = 0; i < terms.length; i++) {
- totalTF = getTotalTermFreq(reader, terms[i].term);
- ts[i] = new TermStats( terms[i].term, terms[i].docFreq, totalTF);
- }
-
- Comparator<TermStats> c = new TotalTermFreqComparatorSortDescending();
- Arrays.sort(ts, c);
-
- return ts;
- }
-
- public static long getTotalTermFreq(IndexReader reader, Term term) throws Exception {
- long totalTF = 0;
- TermDocs td = reader.termDocs(term);
- while (td.next()) {
- totalTF += td.freq();
- }
- return totalTF;
- }
-}
-
-
-final class TermStats {
- public Term term;
- public int docFreq;
- public long totalTermFreq;
-
- public TermStats(Term t, int df) {
- this.term = t;
- this.docFreq = df;
- }
-
- public TermStats(Term t, int df, long tf) {
- this.term = t;
- this.docFreq = df;
- this.totalTermFreq = tf;
- }
-}
-
-
-/**
- * Priority queue for TermStats objects ordered by TermStats.docFreq
- **/
-final class TermInfoWiTFQueue extends PriorityQueue<TermStats> {
- TermInfoWiTFQueue(int size) {
- initialize(size);
- }
-
- @Override
- protected boolean lessThan(TermStats termInfoA,
- TermStats termInfoB) {
- return termInfoA.docFreq < termInfoB.docFreq;
- }
-}
-
-/**
- * Comparator
- *
- * Reverse of normal Comparator. i.e. returns 1 if a.totalTermFreq is less than
- * b.totalTermFreq So we can sort in descending order of totalTermFreq
- */
-final class TotalTermFreqComparatorSortDescending implements Comparator<TermStats> {
-
- public int compare(TermStats a, TermStats b) {
- if (a.totalTermFreq < b.totalTermFreq) {
- return 1;
- } else if (a.totalTermFreq > b.totalTermFreq) {
- return -1;
- } else {
- return 0;
- }
- }
-}
-
fnp / pylucene.git / blobdiff