+++ /dev/null
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.io.Reader;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-
-/**
- * Holds state for inverting all occurrences of a single
- * field in the document. This class doesn't do anything
- * itself; instead, it forwards the tokens produced by
- * analysis to its own consumer
- * (InvertedDocConsumerPerField). It also interacts with an
- * endConsumer (InvertedDocEndConsumerPerField).
- */
-
-final class DocInverterPerField extends DocFieldConsumerPerField {
-
- final private DocInverterPerThread perThread;
- final private FieldInfo fieldInfo;
- final InvertedDocConsumerPerField consumer;
- final InvertedDocEndConsumerPerField endConsumer;
- final DocumentsWriter.DocState docState;
- final FieldInvertState fieldState;
-
- public DocInverterPerField(DocInverterPerThread perThread, FieldInfo fieldInfo) {
- this.perThread = perThread;
- this.fieldInfo = fieldInfo;
- docState = perThread.docState;
- fieldState = perThread.fieldState;
- this.consumer = perThread.consumer.addField(this, fieldInfo);
- this.endConsumer = perThread.endConsumer.addField(this, fieldInfo);
- }
-
- @Override
- void abort() {
- try {
- consumer.abort();
- } finally {
- endConsumer.abort();
- }
- }
-
- @Override
- public void processFields(final Fieldable[] fields,
- final int count) throws IOException {
-
- fieldState.reset(docState.doc.getBoost());
-
- final int maxFieldLength = docState.maxFieldLength;
-
- final boolean doInvert = consumer.start(fields, count);
-
- for(int i=0;i<count;i++) {
-
- final Fieldable field = fields[i];
-
- // TODO FI: this should be "genericized" to querying
- // consumer if it wants to see this particular field
- // tokenized.
- if (field.isIndexed() && doInvert) {
-
- if (i > 0)
- fieldState.position += docState.analyzer == null ? 0 : docState.analyzer.getPositionIncrementGap(fieldInfo.name);
-
- if (!field.isTokenized()) { // un-tokenized field
- String stringValue = field.stringValue();
- final int valueLength = stringValue.length();
- perThread.singleToken.reinit(stringValue, 0, valueLength);
- fieldState.attributeSource = perThread.singleToken;
- consumer.start(field);
-
- boolean success = false;
- try {
- consumer.add();
- success = true;
- } finally {
- if (!success)
- docState.docWriter.setAborting();
- }
- fieldState.offset += valueLength;
- fieldState.length++;
- fieldState.position++;
- } else { // tokenized field
- final TokenStream stream;
- final TokenStream streamValue = field.tokenStreamValue();
-
- if (streamValue != null)
- stream = streamValue;
- else {
- // the field does not have a TokenStream,
- // so we have to obtain one from the analyzer
- final Reader reader; // find or make Reader
- final Reader readerValue = field.readerValue();
-
- if (readerValue != null)
- reader = readerValue;
- else {
- String stringValue = field.stringValue();
- if (stringValue == null)
- throw new IllegalArgumentException("field must have either TokenStream, String or Reader value");
- perThread.stringReader.init(stringValue);
- reader = perThread.stringReader;
- }
-
- // Tokenize field and add to postingTable
- stream = docState.analyzer.reusableTokenStream(fieldInfo.name, reader);
- }
-
- // reset the TokenStream to the first token
- stream.reset();
-
- final int startLength = fieldState.length;
-
- try {
- boolean hasMoreTokens = stream.incrementToken();
-
- fieldState.attributeSource = stream;
-
- OffsetAttribute offsetAttribute = fieldState.attributeSource.addAttribute(OffsetAttribute.class);
- PositionIncrementAttribute posIncrAttribute = fieldState.attributeSource.addAttribute(PositionIncrementAttribute.class);
-
- consumer.start(field);
-
- for(;;) {
-
- // If we hit an exception in stream.next below
- // (which is fairly common, eg if analyzer
- // chokes on a given document), then it's
- // non-aborting and (above) this one document
- // will be marked as deleted, but still
- // consume a docID
-
- if (!hasMoreTokens) break;
-
- final int posIncr = posIncrAttribute.getPositionIncrement();
- fieldState.position += posIncr;
- if (fieldState.position > 0) {
- fieldState.position--;
- }
-
- if (posIncr == 0)
- fieldState.numOverlap++;
-
- boolean success = false;
- try {
- // If we hit an exception in here, we abort
- // all buffered documents since the last
- // flush, on the likelihood that the
- // internal state of the consumer is now
- // corrupt and should not be flushed to a
- // new segment:
- consumer.add();
- success = true;
- } finally {
- if (!success)
- docState.docWriter.setAborting();
- }
- fieldState.position++;
- if (++fieldState.length >= maxFieldLength) {
- if (docState.infoStream != null)
- docState.infoStream.println("maxFieldLength " +maxFieldLength+ " reached for field " + fieldInfo.name + ", ignoring following tokens");
- break;
- }
-
- hasMoreTokens = stream.incrementToken();
- }
- // trigger streams to perform end-of-stream operations
- stream.end();
-
- fieldState.offset += offsetAttribute.endOffset();
- } finally {
- stream.close();
- }
- }
-
- fieldState.offset += docState.analyzer == null ? 0 : docState.analyzer.getOffsetGap(field);
- fieldState.boost *= field.getBoost();
- }
-
- // LUCENE-2387: don't hang onto the field, so GC can
- // reclaim
- fields[i] = null;
- }
-
- consumer.finish();
- endConsumer.finish();
- }
-}
fnp / pylucene.git / blobdiff