--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.StringHelper;
+import org.apache.lucene.util.UnicodeUtil;
+
+import java.io.IOException;
+
+final class TermVectorsWriter {
+
+ private IndexOutput tvx = null, tvd = null, tvf = null;
+ private FieldInfos fieldInfos;
+ final UnicodeUtil.UTF8Result[] utf8Results = new UnicodeUtil.UTF8Result[] {new UnicodeUtil.UTF8Result(),
+ new UnicodeUtil.UTF8Result()};
+
+ public TermVectorsWriter(Directory directory, String segment,
+ FieldInfos fieldInfos) throws IOException {
+ boolean success = false;
+ try {
+ // Open files for TermVector storage
+ tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_INDEX_EXTENSION));
+ tvx.writeInt(TermVectorsReader.FORMAT_CURRENT);
+ tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
+ tvd.writeInt(TermVectorsReader.FORMAT_CURRENT);
+ tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_FIELDS_EXTENSION));
+ tvf.writeInt(TermVectorsReader.FORMAT_CURRENT);
+ success = true;
+ } finally {
+ if (!success) {
+ IOUtils.closeWhileHandlingException(tvx, tvd, tvf);
+ }
+ }
+
+ this.fieldInfos = fieldInfos;
+ }
+
+ /**
+ * Add a complete document specified by all its term vectors. If document has no
+ * term vectors, add value for tvx.
+ *
+ * @param vectors
+ * @throws IOException
+ */
+ public final void addAllDocVectors(TermFreqVector[] vectors) throws IOException {
+
+ tvx.writeLong(tvd.getFilePointer());
+ tvx.writeLong(tvf.getFilePointer());
+
+ if (vectors != null) {
+ final int numFields = vectors.length;
+ tvd.writeVInt(numFields);
+
+ long[] fieldPointers = new long[numFields];
+
+ for (int i=0; i<numFields; i++) {
+ fieldPointers[i] = tvf.getFilePointer();
+
+ final int fieldNumber = fieldInfos.fieldNumber(vectors[i].getField());
+
+ // 1st pass: write field numbers to tvd
+ tvd.writeVInt(fieldNumber);
+
+ final int numTerms = vectors[i].size();
+ tvf.writeVInt(numTerms);
+
+ final TermPositionVector tpVector;
+
+ final byte bits;
+ final boolean storePositions;
+ final boolean storeOffsets;
+
+ if (vectors[i] instanceof TermPositionVector) {
+ // May have positions & offsets
+ tpVector = (TermPositionVector) vectors[i];
+ storePositions = tpVector.size() > 0 && tpVector.getTermPositions(0) != null;
+ storeOffsets = tpVector.size() > 0 && tpVector.getOffsets(0) != null;
+ bits = (byte) ((storePositions ? TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR : 0) +
+ (storeOffsets ? TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR : 0));
+ } else {
+ tpVector = null;
+ bits = 0;
+ storePositions = false;
+ storeOffsets = false;
+ }
+
+ tvf.writeVInt(bits);
+
+ final String[] terms = vectors[i].getTerms();
+ final int[] freqs = vectors[i].getTermFrequencies();
+
+ int utf8Upto = 0;
+ utf8Results[1].length = 0;
+
+ for (int j=0; j<numTerms; j++) {
+
+ UnicodeUtil.UTF16toUTF8(terms[j], 0, terms[j].length(), utf8Results[utf8Upto]);
+
+ int start = StringHelper.bytesDifference(utf8Results[1-utf8Upto].result,
+ utf8Results[1-utf8Upto].length,
+ utf8Results[utf8Upto].result,
+ utf8Results[utf8Upto].length);
+ int length = utf8Results[utf8Upto].length - start;
+ tvf.writeVInt(start); // write shared prefix length
+ tvf.writeVInt(length); // write delta length
+ tvf.writeBytes(utf8Results[utf8Upto].result, start, length); // write delta bytes
+ utf8Upto = 1-utf8Upto;
+
+ final int termFreq = freqs[j];
+
+ tvf.writeVInt(termFreq);
+
+ if (storePositions) {
+ final int[] positions = tpVector.getTermPositions(j);
+ if (positions == null)
+ throw new IllegalStateException("Trying to write positions that are null!");
+ assert positions.length == termFreq;
+
+ // use delta encoding for positions
+ int lastPosition = 0;
+ for(int k=0;k<positions.length;k++) {
+ final int position = positions[k];
+ tvf.writeVInt(position-lastPosition);
+ lastPosition = position;
+ }
+ }
+
+ if (storeOffsets) {
+ final TermVectorOffsetInfo[] offsets = tpVector.getOffsets(j);
+ if (offsets == null)
+ throw new IllegalStateException("Trying to write offsets that are null!");
+ assert offsets.length == termFreq;
+
+ // use delta encoding for offsets
+ int lastEndOffset = 0;
+ for(int k=0;k<offsets.length;k++) {
+ final int startOffset = offsets[k].getStartOffset();
+ final int endOffset = offsets[k].getEndOffset();
+ tvf.writeVInt(startOffset-lastEndOffset);
+ tvf.writeVInt(endOffset-startOffset);
+ lastEndOffset = endOffset;
+ }
+ }
+ }
+ }
+
+ // 2nd pass: write field pointers to tvd
+ if (numFields > 1) {
+ long lastFieldPointer = fieldPointers[0];
+ for (int i=1; i<numFields; i++) {
+ final long fieldPointer = fieldPointers[i];
+ tvd.writeVLong(fieldPointer-lastFieldPointer);
+ lastFieldPointer = fieldPointer;
+ }
+ }
+ } else
+ tvd.writeVInt(0);
+ }
+
+ /**
+ * Do a bulk copy of numDocs documents from reader to our
+ * streams. This is used to expedite merging, if the
+ * field numbers are congruent.
+ */
+ final void addRawDocuments(TermVectorsReader reader, int[] tvdLengths, int[] tvfLengths, int numDocs) throws IOException {
+ long tvdPosition = tvd.getFilePointer();
+ long tvfPosition = tvf.getFilePointer();
+ long tvdStart = tvdPosition;
+ long tvfStart = tvfPosition;
+ for(int i=0;i<numDocs;i++) {
+ tvx.writeLong(tvdPosition);
+ tvdPosition += tvdLengths[i];
+ tvx.writeLong(tvfPosition);
+ tvfPosition += tvfLengths[i];
+ }
+ tvd.copyBytes(reader.getTvdStream(), tvdPosition-tvdStart);
+ tvf.copyBytes(reader.getTvfStream(), tvfPosition-tvfStart);
+ assert tvd.getFilePointer() == tvdPosition;
+ assert tvf.getFilePointer() == tvfPosition;
+ }
+
+ /** Close all streams. */
+ final void close() throws IOException {
+ // make an effort to close all streams we can but remember and re-throw
+ // the first exception encountered in this process
+ IOUtils.close(tvx, tvd, tvf);
+ }
+}
fnp / pylucene.git / blobdiff