+++ /dev/null
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.lucene.index;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.logging.Logger;
-
-import org.apache.lucene.analysis.WhitespaceAnalyzer;
-import org.apache.lucene.document.*;
-import org.apache.lucene.index.IndexWriter; // javadocs
-import org.apache.lucene.store.*;
-import org.apache.lucene.util.Version;
-
-/** Sort an index by document importance factor. Higher scoring documents are
- * assigned smaller document numbers. Document weights are obtained from a
- * specified field, which has to be single-valued and stored, with string value
- * that represents a float number. Stored fields in the output index remain
- * consistent, i.e. both stored fields and postings are renumbered in sync.
- *
- * <p><b>NOTE</b>: this tool is unaware of documents added
- * atomically via {@link IndexWriter#addDocuments} or {@link
- * IndexWriter#updateDocuments}, which means it can easily
- * break up such document groups.
- */
-public class IndexSorter {
- private static final Logger LOG = Logger.getLogger(IndexSorter.class.getName());
-
- private static class PostingMap implements Comparable<PostingMap> {
- private int newDoc;
- private long offset;
-
- public int compareTo(PostingMap pm) { // order by newDoc id
- return this.newDoc - pm.newDoc;
- }
- }
-
- private static class SortedTermPositions implements TermPositions {
- private TermPositions original;
- private int[] oldToNew;
-
- private int docFreq;
-
- private PostingMap[] postingMaps = new PostingMap[0];
- private int pointer;
-
- private int freq;
- private int position;
-
- private static final String TEMP_FILE = "temp";
- private final RAMDirectory tempDir = new RAMDirectory();
- private RAMOutputStream out;
- private IndexInput in;
-
- public SortedTermPositions(TermPositions original, int[] oldToNew) {
- this.original = original;
- this.oldToNew = oldToNew;
- try {
- out = (RAMOutputStream)tempDir.createOutput(TEMP_FILE);
- } catch (IOException ioe) {
- LOG.warning("Error creating temporary output: " + ioe);
- }
- }
-
- public void seek(Term term) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- public void seek(TermEnum terms) throws IOException {
- original.seek(terms);
-
- docFreq = terms.docFreq();
- pointer = -1;
-
- if (docFreq > postingMaps.length) { // grow postingsMap
- PostingMap[] newMap = new PostingMap[docFreq];
- System.arraycopy(postingMaps, 0, newMap, 0, postingMaps.length);
- for (int i = postingMaps.length; i < docFreq; i++) {
- newMap[i] = new PostingMap();
- }
- postingMaps = newMap;
- }
-
- out.reset();
-
- int i = 0;
- while (original.next()) {
- PostingMap map = postingMaps[i++];
- map.newDoc = oldToNew[original.doc()]; // remap the newDoc id
- map.offset = out.getFilePointer(); // save pointer to buffer
-
- final int tf = original.freq(); // buffer tf & positions
- out.writeVInt(tf);
- int prevPosition = 0;
- for (int j = tf; j > 0; j--) { // delta encode positions
- int p = original.nextPosition();
- out.writeVInt(p - prevPosition);
- prevPosition = p;
- }
- }
- out.flush();
- docFreq = i; // allow for deletions
-
- Arrays.sort(postingMaps, 0, docFreq); // resort by mapped doc ids
-
- // NOTE: this might be substantially faster if RAMInputStream were public
- // and supported a reset() operation.
- in = tempDir.openInput(TEMP_FILE);
- }
-
- public boolean next() throws IOException {
- pointer++;
- if (pointer < docFreq) {
- in.seek(postingMaps[pointer].offset);
- freq = in.readVInt();
- position = 0;
- return true;
- }
- return false;
- }
-
- public int doc() { return postingMaps[pointer].newDoc; }
- public int freq() { return freq; }
-
- public int nextPosition() throws IOException {
- int positionIncrement = in.readVInt();
- position += positionIncrement;
- return position;
- }
-
- public int read(int[] docs, int[] freqs) {
- throw new UnsupportedOperationException();
- }
- public boolean skipTo(int target) {
- throw new UnsupportedOperationException();
- }
-
- public byte[] getPayload(byte[] data, int offset) throws IOException {
- return null;
- }
-
- public int getPayloadLength() {
- return 0;
- }
-
- public boolean isPayloadAvailable() {
- return false;
- }
-
- public void close() throws IOException {
- original.close();
- }
-
- }
-
- private static class SortingReader extends FilterIndexReader {
-
- private int[] oldToNew;
- private int[] newToOld;
-
- public SortingReader(IndexReader oldReader, int[] oldToNew) {
- super(oldReader);
- this.oldToNew = oldToNew;
-
- this.newToOld = new int[oldReader.maxDoc()];
- int oldDoc = 0;
- while (oldDoc < oldToNew.length) {
- int newDoc = oldToNew[oldDoc];
- if (newDoc != -1) {
- newToOld[newDoc] = oldDoc;
- }
- oldDoc++;
- }
- }
-
- @Override
- public IndexReader[] getSequentialSubReaders() {
- return null;
- }
-
- @Override
- public Document document(int n) throws IOException {
- return document(n, null);
- }
-
- @Override
- public Document document(int n, FieldSelector fieldSelector)
- throws CorruptIndexException, IOException {
- return super.document(newToOld[n], fieldSelector);
- }
-
- @Override
- public boolean isDeleted(int n) {
- return false;
- }
-
- @Override
- public byte[] norms(String f) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void norms(String f, byte[] norms, int offset) throws IOException {
- byte[] oldNorms = super.norms(f);
- int oldDoc = 0;
- while (oldDoc < oldNorms.length) {
- int newDoc = oldToNew[oldDoc];
- if (newDoc != -1) {
- norms[newDoc] = oldNorms[oldDoc];
- }
- oldDoc++;
- }
- }
-
- @Override
- protected void doSetNorm(int d, String f, byte b) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public TermDocs termDocs() throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public TermPositions termPositions() throws IOException {
- return new SortedTermPositions(super.termPositions(), oldToNew);
- }
-
- @Override
- public TermFreqVector[] getTermFreqVectors(int docNumber)
- throws IOException {
- return super.getTermFreqVectors(newToOld[docNumber]);
- }
-
- @Override
- protected void doDelete(int n) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- }
-
- private static class DocScore implements Comparable<DocScore> {
- private int oldDoc;
- private float score;
-
- public int compareTo(DocScore that) { // order by score, oldDoc
- if (this.score == that.score) {
- return this.oldDoc - that.oldDoc;
- } else {
- return this.score < that.score ? 1 : -1 ;
- }
- }
-
- @Override
- public String toString() {
- return "oldDoc=" + oldDoc + ",score=" + score;
- }
- }
-
- public IndexSorter() {
-
- }
-
- public void sort(Directory input, Directory output, String field) throws IOException {
- LOG.info("IndexSorter: starting.");
- long start = System.currentTimeMillis();
- IndexReader reader = IndexReader.open(input, true);
-
- SortingReader sorter = new SortingReader(reader, oldToNew(reader, field));
- IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_31, new WhitespaceAnalyzer(Version.LUCENE_31));
- IndexWriter writer = new IndexWriter(output, cfg);
- writer.addIndexes(new IndexReader[] { sorter });
- writer.close();
- long end = System.currentTimeMillis();
- LOG.info("IndexSorter: done, " + (end - start)
- + " total milliseconds");
- }
-
- private static int[] oldToNew(IndexReader reader, String field) throws IOException {
- int readerMax = reader.maxDoc();
- DocScore[] newToOld = new DocScore[readerMax];
- FieldSelector fSel = new MapFieldSelector(field);
-
- for (int oldDoc = 0; oldDoc < readerMax; oldDoc++) {
- float score;
- if (reader.isDeleted(oldDoc)) {
- score = 0.0f;
- } else {
- Document d = reader.document(oldDoc, fSel);
- try {
- score = Float.parseFloat(d.get(field));
- } catch (Exception e) {
- score = 0.0f;
- }
- }
- DocScore docScore = new DocScore();
- docScore.oldDoc = oldDoc;
- docScore.score = score;
- newToOld[oldDoc] = docScore;
- }
- Arrays.sort(newToOld);
-
- int[] oldToNew = new int[readerMax];
- for (int newDoc = 0; newDoc < readerMax; newDoc++) {
- DocScore docScore = newToOld[newDoc];
- oldToNew[docScore.oldDoc] = newDoc;
- }
- return oldToNew;
- }
-
- /** */
- public static void main(String[] args) throws Exception {
- Directory input, output;
- String field;
-
- String usage = "IndexSorter <input> <output> <field>";
-
- if (args.length < 3) {
- System.err.println("Usage: " + usage);
- System.exit(-1);
- }
-
- input = FSDirectory.open(new File(args[0]));
- File out = new File(args[1]);
- if (!out.exists()) out.mkdirs();
- output = FSDirectory.open(out);
- field = args[2];
- IndexSorter sorter = new IndexSorter();
- try {
- sorter.sort(input, output, field);
- } catch (Exception e) {
- LOG.warning("IndexSorter: " + e);
- }
- }
-}
fnp / pylucene.git / blobdiff