+++ /dev/null
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util.RamUsageEstimator;
-
-final class TermVectorsTermsWriterPerField extends TermsHashConsumerPerField {
-
- final TermVectorsTermsWriterPerThread perThread;
- final TermsHashPerField termsHashPerField;
- final TermVectorsTermsWriter termsWriter;
- final FieldInfo fieldInfo;
- final DocumentsWriter.DocState docState;
- final FieldInvertState fieldState;
-
- boolean doVectors;
- boolean doVectorPositions;
- boolean doVectorOffsets;
-
- int maxNumPostings;
- OffsetAttribute offsetAttribute = null;
-
- public TermVectorsTermsWriterPerField(TermsHashPerField termsHashPerField, TermVectorsTermsWriterPerThread perThread, FieldInfo fieldInfo) {
- this.termsHashPerField = termsHashPerField;
- this.perThread = perThread;
- this.termsWriter = perThread.termsWriter;
- this.fieldInfo = fieldInfo;
- docState = termsHashPerField.docState;
- fieldState = termsHashPerField.fieldState;
- }
-
- @Override
- int getStreamCount() {
- return 2;
- }
-
- @Override
- boolean start(Fieldable[] fields, int count) {
- doVectors = false;
- doVectorPositions = false;
- doVectorOffsets = false;
-
- for(int i=0;i<count;i++) {
- Fieldable field = fields[i];
- if (field.isIndexed() && field.isTermVectorStored()) {
- doVectors = true;
- doVectorPositions |= field.isStorePositionWithTermVector();
- doVectorOffsets |= field.isStoreOffsetWithTermVector();
- }
- }
-
- if (doVectors) {
- if (perThread.doc == null) {
- perThread.doc = termsWriter.getPerDoc();
- perThread.doc.docID = docState.docID;
- assert perThread.doc.numVectorFields == 0;
- assert 0 == perThread.doc.perDocTvf.length();
- assert 0 == perThread.doc.perDocTvf.getFilePointer();
- }
-
- assert perThread.doc.docID == docState.docID;
-
- if (termsHashPerField.numPostings != 0) {
- // Only necessary if previous doc hit a
- // non-aborting exception while writing vectors in
- // this field:
- termsHashPerField.reset();
- perThread.termsHashPerThread.reset(false);
- }
- }
-
- // TODO: only if needed for performance
- //perThread.postingsCount = 0;
-
- return doVectors;
- }
-
- public void abort() {}
-
- /** Called once per field per document if term vectors
- * are enabled, to write the vectors to
- * RAMOutputStream, which is then quickly flushed to
- * the real term vectors files in the Directory. */
- @Override
- void finish() throws IOException {
-
- assert docState.testPoint("TermVectorsTermsWriterPerField.finish start");
-
- final int numPostings = termsHashPerField.numPostings;
-
- assert numPostings >= 0;
-
- if (!doVectors || numPostings == 0)
- return;
-
- if (numPostings > maxNumPostings)
- maxNumPostings = numPostings;
-
- final IndexOutput tvf = perThread.doc.perDocTvf;
-
- // This is called once, after inverting all occurrences
- // of a given field in the doc. At this point we flush
- // our hash into the DocWriter.
-
- assert fieldInfo.storeTermVector;
- assert perThread.vectorFieldsInOrder(fieldInfo);
-
- perThread.doc.addField(termsHashPerField.fieldInfo.number);
- TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
-
- final int[] termIDs = termsHashPerField.sortPostings();
-
- tvf.writeVInt(numPostings);
- byte bits = 0x0;
- if (doVectorPositions)
- bits |= TermVectorsReader.STORE_POSITIONS_WITH_TERMVECTOR;
- if (doVectorOffsets)
- bits |= TermVectorsReader.STORE_OFFSET_WITH_TERMVECTOR;
- tvf.writeByte(bits);
-
- int encoderUpto = 0;
- int lastTermBytesCount = 0;
-
- final ByteSliceReader reader = perThread.vectorSliceReader;
- final char[][] charBuffers = perThread.termsHashPerThread.charPool.buffers;
- for(int j=0;j<numPostings;j++) {
- final int termID = termIDs[j];
- final int freq = postings.freqs[termID];
-
- final char[] text2 = charBuffers[postings.textStarts[termID] >> DocumentsWriter.CHAR_BLOCK_SHIFT];
- final int start2 = postings.textStarts[termID] & DocumentsWriter.CHAR_BLOCK_MASK;
-
- // We swap between two encoders to save copying
- // last Term's byte array
- final UnicodeUtil.UTF8Result utf8Result = perThread.utf8Results[encoderUpto];
-
- // TODO: we could do this incrementally
- UnicodeUtil.UTF16toUTF8(text2, start2, utf8Result);
- final int termBytesCount = utf8Result.length;
-
- // TODO: UTF16toUTF8 could tell us this prefix
- // Compute common prefix between last term and
- // this term
- int prefix = 0;
- if (j > 0) {
- final byte[] lastTermBytes = perThread.utf8Results[1-encoderUpto].result;
- final byte[] termBytes = perThread.utf8Results[encoderUpto].result;
- while(prefix < lastTermBytesCount && prefix < termBytesCount) {
- if (lastTermBytes[prefix] != termBytes[prefix])
- break;
- prefix++;
- }
- }
- encoderUpto = 1-encoderUpto;
- lastTermBytesCount = termBytesCount;
-
- final int suffix = termBytesCount - prefix;
- tvf.writeVInt(prefix);
- tvf.writeVInt(suffix);
- tvf.writeBytes(utf8Result.result, prefix, suffix);
- tvf.writeVInt(freq);
-
- if (doVectorPositions) {
- termsHashPerField.initReader(reader, termID, 0);
- reader.writeTo(tvf);
- }
-
- if (doVectorOffsets) {
- termsHashPerField.initReader(reader, termID, 1);
- reader.writeTo(tvf);
- }
- }
-
- termsHashPerField.reset();
-
- // NOTE: we clear, per-field, at the thread level,
- // because term vectors fully write themselves on each
- // field; this saves RAM (eg if large doc has two large
- // fields w/ term vectors on) because we recycle/reuse
- // all RAM after each field:
- perThread.termsHashPerThread.reset(false);
- }
-
- void shrinkHash() {
- termsHashPerField.shrinkHash(maxNumPostings);
- maxNumPostings = 0;
- }
-
- @Override
- void start(Fieldable f) {
- if (doVectorOffsets) {
- offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
- } else {
- offsetAttribute = null;
- }
- }
-
- @Override
- void newTerm(final int termID) {
-
- assert docState.testPoint("TermVectorsTermsWriterPerField.newTerm start");
-
- TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
-
- postings.freqs[termID] = 1;
-
- if (doVectorOffsets) {
- int startOffset = fieldState.offset + offsetAttribute.startOffset();
- int endOffset = fieldState.offset + offsetAttribute.endOffset();
-
- termsHashPerField.writeVInt(1, startOffset);
- termsHashPerField.writeVInt(1, endOffset - startOffset);
- postings.lastOffsets[termID] = endOffset;
- }
-
- if (doVectorPositions) {
- termsHashPerField.writeVInt(0, fieldState.position);
- postings.lastPositions[termID] = fieldState.position;
- }
- }
-
- @Override
- void addTerm(final int termID) {
-
- assert docState.testPoint("TermVectorsTermsWriterPerField.addTerm start");
-
- TermVectorsPostingsArray postings = (TermVectorsPostingsArray) termsHashPerField.postingsArray;
-
- postings.freqs[termID]++;
-
- if (doVectorOffsets) {
- int startOffset = fieldState.offset + offsetAttribute.startOffset();
- int endOffset = fieldState.offset + offsetAttribute.endOffset();
-
- termsHashPerField.writeVInt(1, startOffset - postings.lastOffsets[termID]);
- termsHashPerField.writeVInt(1, endOffset - startOffset);
- postings.lastOffsets[termID] = endOffset;
- }
-
- if (doVectorPositions) {
- termsHashPerField.writeVInt(0, fieldState.position - postings.lastPositions[termID]);
- postings.lastPositions[termID] = fieldState.position;
- }
- }
-
- @Override
- void skippingLongTerm() {}
-
- @Override
- ParallelPostingsArray createPostingsArray(int size) {
- return new TermVectorsPostingsArray(size);
- }
-
- static final class TermVectorsPostingsArray extends ParallelPostingsArray {
- public TermVectorsPostingsArray(int size) {
- super(size);
- freqs = new int[size];
- lastOffsets = new int[size];
- lastPositions = new int[size];
- }
-
- int[] freqs; // How many times this term occurred in the current doc
- int[] lastOffsets; // Last offset we saw
- int[] lastPositions; // Last position where this term occurred
-
- @Override
- ParallelPostingsArray newInstance(int size) {
- return new TermVectorsPostingsArray(size);
- }
-
- @Override
- void copyTo(ParallelPostingsArray toArray, int numToCopy) {
- assert toArray instanceof TermVectorsPostingsArray;
- TermVectorsPostingsArray to = (TermVectorsPostingsArray) toArray;
-
- super.copyTo(toArray, numToCopy);
-
- System.arraycopy(freqs, 0, to.freqs, 0, size);
- System.arraycopy(lastOffsets, 0, to.lastOffsets, 0, size);
- System.arraycopy(lastPositions, 0, to.lastPositions, 0, size);
- }
-
- @Override
- int bytesPerPosting() {
- return super.bytesPerPosting() + 3 * RamUsageEstimator.NUM_BYTES_INT;
- }
- }
-}
fnp / pylucene.git / blobdiff