+++ /dev/null
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.util.StringHelper;
-import org.apache.lucene.util.UnicodeUtil;
-
-import java.io.IOException;
-
-final class TermVectorsWriter {
-
- private IndexOutput tvx = null, tvd = null, tvf = null;
- private FieldInfos fieldInfos;
- final UnicodeUtil.UTF8Result[] utf8Results = new UnicodeUtil.UTF8Result[] {new UnicodeUtil.UTF8Result(),
- new UnicodeUtil.UTF8Result()};
-
- public TermVectorsWriter(Directory directory, String segment,
- FieldInfos fieldInfos) throws IOException {
- boolean success = false;
- try {
- // Open files for TermVector storage
- tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_INDEX_EXTENSION));
- tvx.writeInt(TermVectorsReader.FORMAT_CURRENT);
- tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
- tvd.writeInt(TermVectorsReader.FORMAT_CURRENT);
- tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_FIELDS_EXTENSION));
- tvf.writeInt(TermVectorsReader.FORMAT_CURRENT);
- success = true;
- } finally {
- if (!success) {
- IOUtils.closeWhileHandlingException(tvx, tvd, tvf);
- }
- }
-
- this.fieldInfos = fieldInfos;
- }
-
- /**
- * Add a complete document specified by all its term vectors. If document has no
- * term vectors, add value for tvx.
- *
- * @param vectors
- * @throws IOException
- */
- public final void addAllDocVectors(TermFreqVector[] vectors) throws IOException {
-
- tvx.writeLong(tvd.getFilePointer());
- tvx.writeLong(tvf.getFilePointer());
-
- if (vectors != null) {
- final int numFields = vectors.length;
- tvd.writeVInt(numFields);
-
- long[] fieldPointers = new long[numFields];
-
- for (int i=0; i<numFields; i++) {
- fieldPointers[i] = tvf.getFilePointer();
-
- final int fieldNumber = fieldInfos.fieldNumber(vectors[i].getField());
-
- // 1st pass: write field numbers to tvd
- tvd.writeVInt(fieldNumber);
-
- final int numTerms = vectors[i].size();
- tvf.writeVInt(numTerms);
-
- final TermPositionVector tpVector;
-
- final byte bits;
- final boolean storePositions;
- final boolean storeOffsets;
-
- if (vectors[i] instanceof TermPositionVector) {
- // May have positions & offsets
- tpVector = (TermPositionVector) vectors[i];
- storePositions = tpVector.size() > 0 && tpVector.getTermPositions(0) != null;
- storeOffsets = tpVector.size() > 0 && tpVector.getOffsets(0) != null;
- bits = (byte) ((storePositions ? TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR : 0) +
- (storeOffsets ? TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR : 0));
- } else {
- tpVector = null;
- bits = 0;
- storePositions = false;
- storeOffsets = false;
- }
-
- tvf.writeVInt(bits);
-
- final String[] terms = vectors[i].getTerms();
- final int[] freqs = vectors[i].getTermFrequencies();
-
- int utf8Upto = 0;
- utf8Results[1].length = 0;
-
- for (int j=0; j<numTerms; j++) {
-
- UnicodeUtil.UTF16toUTF8(terms[j], 0, terms[j].length(), utf8Results[utf8Upto]);
-
- int start = StringHelper.bytesDifference(utf8Results[1-utf8Upto].result,
- utf8Results[1-utf8Upto].length,
- utf8Results[utf8Upto].result,
- utf8Results[utf8Upto].length);
- int length = utf8Results[utf8Upto].length - start;
- tvf.writeVInt(start); // write shared prefix length
- tvf.writeVInt(length); // write delta length
- tvf.writeBytes(utf8Results[utf8Upto].result, start, length); // write delta bytes
- utf8Upto = 1-utf8Upto;
-
- final int termFreq = freqs[j];
-
- tvf.writeVInt(termFreq);
-
- if (storePositions) {
- final int[] positions = tpVector.getTermPositions(j);
- if (positions == null)
- throw new IllegalStateException("Trying to write positions that are null!");
- assert positions.length == termFreq;
-
- // use delta encoding for positions
- int lastPosition = 0;
- for(int k=0;k<positions.length;k++) {
- final int position = positions[k];
- tvf.writeVInt(position-lastPosition);
- lastPosition = position;
- }
- }
-
- if (storeOffsets) {
- final TermVectorOffsetInfo[] offsets = tpVector.getOffsets(j);
- if (offsets == null)
- throw new IllegalStateException("Trying to write offsets that are null!");
- assert offsets.length == termFreq;
-
- // use delta encoding for offsets
- int lastEndOffset = 0;
- for(int k=0;k<offsets.length;k++) {
- final int startOffset = offsets[k].getStartOffset();
- final int endOffset = offsets[k].getEndOffset();
- tvf.writeVInt(startOffset-lastEndOffset);
- tvf.writeVInt(endOffset-startOffset);
- lastEndOffset = endOffset;
- }
- }
- }
- }
-
- // 2nd pass: write field pointers to tvd
- if (numFields > 1) {
- long lastFieldPointer = fieldPointers[0];
- for (int i=1; i<numFields; i++) {
- final long fieldPointer = fieldPointers[i];
- tvd.writeVLong(fieldPointer-lastFieldPointer);
- lastFieldPointer = fieldPointer;
- }
- }
- } else
- tvd.writeVInt(0);
- }
-
- /**
- * Do a bulk copy of numDocs documents from reader to our
- * streams. This is used to expedite merging, if the
- * field numbers are congruent.
- */
- final void addRawDocuments(TermVectorsReader reader, int[] tvdLengths, int[] tvfLengths, int numDocs) throws IOException {
- long tvdPosition = tvd.getFilePointer();
- long tvfPosition = tvf.getFilePointer();
- long tvdStart = tvdPosition;
- long tvfStart = tvfPosition;
- for(int i=0;i<numDocs;i++) {
- tvx.writeLong(tvdPosition);
- tvdPosition += tvdLengths[i];
- tvx.writeLong(tvfPosition);
- tvfPosition += tvfLengths[i];
- }
- tvd.copyBytes(reader.getTvdStream(), tvdPosition-tvdStart);
- tvf.copyBytes(reader.getTvfStream(), tvfPosition-tvfStart);
- assert tvd.getFilePointer() == tvdPosition;
- assert tvf.getFilePointer() == tvfPosition;
- }
-
- /** Close all streams. */
- final void close() throws IOException {
- // make an effort to close all streams we can but remember and re-throw
- // the first exception encountered in this process
- IOUtils.close(tvx, tvd, tvf);
- }
-}
fnp / pylucene.git / blobdiff