+++ /dev/null
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.util.RamUsageEstimator;
-
-// TODO: break into separate freq and prox writers as
-// codecs; make separate container (tii/tis/skip/*) that can
-// be configured as any number of files 1..N
-final class FreqProxTermsWriterPerField extends TermsHashConsumerPerField implements Comparable<FreqProxTermsWriterPerField> {
-
- final FreqProxTermsWriterPerThread perThread;
- final TermsHashPerField termsHashPerField;
- final FieldInfo fieldInfo;
- final DocumentsWriter.DocState docState;
- final FieldInvertState fieldState;
- IndexOptions indexOptions;
- PayloadAttribute payloadAttribute;
-
- public FreqProxTermsWriterPerField(TermsHashPerField termsHashPerField, FreqProxTermsWriterPerThread perThread, FieldInfo fieldInfo) {
- this.termsHashPerField = termsHashPerField;
- this.perThread = perThread;
- this.fieldInfo = fieldInfo;
- docState = termsHashPerField.docState;
- fieldState = termsHashPerField.fieldState;
- indexOptions = fieldInfo.indexOptions;
- }
-
- @Override
- int getStreamCount() {
- if (fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS)
- return 1;
- else
- return 2;
- }
-
- @Override
- void finish() {}
-
- boolean hasPayloads;
-
- @Override
- void skippingLongTerm() throws IOException {}
-
- public int compareTo(FreqProxTermsWriterPerField other) {
- return fieldInfo.name.compareTo(other.fieldInfo.name);
- }
-
- void reset() {
- // Record, up front, whether our in-RAM format will be
- // with or without term freqs:
- indexOptions = fieldInfo.indexOptions;
- payloadAttribute = null;
- }
-
- @Override
- boolean start(Fieldable[] fields, int count) {
- for(int i=0;i<count;i++)
- if (fields[i].isIndexed())
- return true;
- return false;
- }
-
- @Override
- void start(Fieldable f) {
- if (fieldState.attributeSource.hasAttribute(PayloadAttribute.class)) {
- payloadAttribute = fieldState.attributeSource.getAttribute(PayloadAttribute.class);
- } else {
- payloadAttribute = null;
- }
- }
-
- void writeProx(final int termID, int proxCode) {
- final Payload payload;
- if (payloadAttribute == null) {
- payload = null;
- } else {
- payload = payloadAttribute.getPayload();
- }
-
- if (payload != null && payload.length > 0) {
- termsHashPerField.writeVInt(1, (proxCode<<1)|1);
- termsHashPerField.writeVInt(1, payload.length);
- termsHashPerField.writeBytes(1, payload.data, payload.offset, payload.length);
- hasPayloads = true;
- } else
- termsHashPerField.writeVInt(1, proxCode<<1);
-
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
- postings.lastPositions[termID] = fieldState.position;
-
- }
-
- @Override
- void newTerm(final int termID) {
- // First time we're seeing this term since the last
- // flush
- assert docState.testPoint("FreqProxTermsWriterPerField.newTerm start");
-
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
- postings.lastDocIDs[termID] = docState.docID;
- if (indexOptions == IndexOptions.DOCS_ONLY) {
- postings.lastDocCodes[termID] = docState.docID;
- } else {
- postings.lastDocCodes[termID] = docState.docID << 1;
- postings.docFreqs[termID] = 1;
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- writeProx(termID, fieldState.position);
- }
- }
- fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
- fieldState.uniqueTermCount++;
- }
-
- @Override
- void addTerm(final int termID) {
-
- assert docState.testPoint("FreqProxTermsWriterPerField.addTerm start");
-
- FreqProxPostingsArray postings = (FreqProxPostingsArray) termsHashPerField.postingsArray;
-
- assert indexOptions == IndexOptions.DOCS_ONLY || postings.docFreqs[termID] > 0;
-
- if (indexOptions == IndexOptions.DOCS_ONLY) {
- if (docState.docID != postings.lastDocIDs[termID]) {
- assert docState.docID > postings.lastDocIDs[termID];
- termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]);
- postings.lastDocCodes[termID] = docState.docID - postings.lastDocIDs[termID];
- postings.lastDocIDs[termID] = docState.docID;
- fieldState.uniqueTermCount++;
- }
- } else {
- if (docState.docID != postings.lastDocIDs[termID]) {
- assert docState.docID > postings.lastDocIDs[termID];
- // Term not yet seen in the current doc but previously
- // seen in other doc(s) since the last flush
-
- // Now that we know doc freq for previous doc,
- // write it & lastDocCode
- if (1 == postings.docFreqs[termID])
- termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]|1);
- else {
- termsHashPerField.writeVInt(0, postings.lastDocCodes[termID]);
- termsHashPerField.writeVInt(0, postings.docFreqs[termID]);
- }
- postings.docFreqs[termID] = 1;
- fieldState.maxTermFrequency = Math.max(1, fieldState.maxTermFrequency);
- postings.lastDocCodes[termID] = (docState.docID - postings.lastDocIDs[termID]) << 1;
- postings.lastDocIDs[termID] = docState.docID;
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- writeProx(termID, fieldState.position);
- }
- fieldState.uniqueTermCount++;
- } else {
- fieldState.maxTermFrequency = Math.max(fieldState.maxTermFrequency, ++postings.docFreqs[termID]);
- if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
- writeProx(termID, fieldState.position-postings.lastPositions[termID]);
- }
- }
- }
- }
-
- @Override
- ParallelPostingsArray createPostingsArray(int size) {
- return new FreqProxPostingsArray(size);
- }
-
- static final class FreqProxPostingsArray extends ParallelPostingsArray {
- public FreqProxPostingsArray(int size) {
- super(size);
- docFreqs = new int[size];
- lastDocIDs = new int[size];
- lastDocCodes = new int[size];
- lastPositions = new int[size];
- }
-
- int docFreqs[]; // # times this term occurs in the current doc
- int lastDocIDs[]; // Last docID where this term occurred
- int lastDocCodes[]; // Code for prior doc
- int lastPositions[]; // Last position where this term occurred
-
- @Override
- ParallelPostingsArray newInstance(int size) {
- return new FreqProxPostingsArray(size);
- }
-
- @Override
- void copyTo(ParallelPostingsArray toArray, int numToCopy) {
- assert toArray instanceof FreqProxPostingsArray;
- FreqProxPostingsArray to = (FreqProxPostingsArray) toArray;
-
- super.copyTo(toArray, numToCopy);
-
- System.arraycopy(docFreqs, 0, to.docFreqs, 0, numToCopy);
- System.arraycopy(lastDocIDs, 0, to.lastDocIDs, 0, numToCopy);
- System.arraycopy(lastDocCodes, 0, to.lastDocCodes, 0, numToCopy);
- System.arraycopy(lastPositions, 0, to.lastPositions, 0, numToCopy);
- }
-
- @Override
- int bytesPerPosting() {
- return ParallelPostingsArray.BYTES_PER_POSTING + 4 * RamUsageEstimator.NUM_BYTES_INT;
- }
- }
-
- public void abort() {}
-}
-
fnp / pylucene.git / blobdiff