--- /dev/null
+/*
+ * Created on 28-Oct-2004
+ */
+package org.apache.lucene.search.highlight;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Comparator;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.index.TermPositionVector;
+import org.apache.lucene.index.TermVectorOffsetInfo;
+import org.apache.lucene.util.ArrayUtil;
+
+/**
+ * Hides implementation issues associated with obtaining a TokenStream for use
+ * with the higlighter - can obtain from TermFreqVectors with offsets and
+ * (optionally) positions or from Analyzer class reparsing the stored content.
+ */
+public class TokenSources {
+ /**
+ * A convenience method that tries to first get a TermPositionVector for the
+ * specified docId, then, falls back to using the passed in
+ * {@link org.apache.lucene.document.Document} to retrieve the TokenStream.
+ * This is useful when you already have the document, but would prefer to use
+ * the vector first.
+ *
+ * @param reader The {@link org.apache.lucene.index.IndexReader} to use to try
+ * and get the vector from
+ * @param docId The docId to retrieve.
+ * @param field The field to retrieve on the document
+ * @param doc The document to fall back on
+ * @param analyzer The analyzer to use for creating the TokenStream if the
+ * vector doesn't exist
+ * @return The {@link org.apache.lucene.analysis.TokenStream} for the
+ * {@link org.apache.lucene.document.Fieldable} on the
+ * {@link org.apache.lucene.document.Document}
+ * @throws IOException if there was an error loading
+ */
+ public static TokenStream getAnyTokenStream(IndexReader reader, int docId,
+ String field, Document doc, Analyzer analyzer) throws IOException {
+ TokenStream ts = null;
+
+ TermFreqVector tfv = reader.getTermFreqVector(docId, field);
+ if (tfv != null) {
+ if (tfv instanceof TermPositionVector) {
+ ts = getTokenStream((TermPositionVector) tfv);
+ }
+ }
+ // No token info stored so fall back to analyzing raw content
+ if (ts == null) {
+ ts = getTokenStream(doc, field, analyzer);
+ }
+ return ts;
+ }
+
+ /**
+ * A convenience method that tries a number of approaches to getting a token
+ * stream. The cost of finding there are no termVectors in the index is
+ * minimal (1000 invocations still registers 0 ms). So this "lazy" (flexible?)
+ * approach to coding is probably acceptable
+ *
+ * @param reader
+ * @param docId
+ * @param field
+ * @param analyzer
+ * @return null if field not stored correctly
+ * @throws IOException
+ */
+ public static TokenStream getAnyTokenStream(IndexReader reader, int docId,
+ String field, Analyzer analyzer) throws IOException {
+ TokenStream ts = null;
+
+ TermFreqVector tfv = reader.getTermFreqVector(docId, field);
+ if (tfv != null) {
+ if (tfv instanceof TermPositionVector) {
+ ts = getTokenStream((TermPositionVector) tfv);
+ }
+ }
+ // No token info stored so fall back to analyzing raw content
+ if (ts == null) {
+ ts = getTokenStream(reader, docId, field, analyzer);
+ }
+ return ts;
+ }
+
+ public static TokenStream getTokenStream(TermPositionVector tpv) {
+ // assumes the worst and makes no assumptions about token position
+ // sequences.
+ return getTokenStream(tpv, false);
+ }
+
+ /**
+ * Low level api. Returns a token stream or null if no offset info available
+ * in index. This can be used to feed the highlighter with a pre-parsed token
+ * stream
+ *
+ * In my tests the speeds to recreate 1000 token streams using this method
+ * are: - with TermVector offset only data stored - 420 milliseconds - with
+ * TermVector offset AND position data stored - 271 milliseconds (nb timings
+ * for TermVector with position data are based on a tokenizer with contiguous
+ * positions - no overlaps or gaps) The cost of not using TermPositionVector
+ * to store pre-parsed content and using an analyzer to re-parse the original
+ * content: - reanalyzing the original content - 980 milliseconds
+ *
+ * The re-analyze timings will typically vary depending on - 1) The complexity
+ * of the analyzer code (timings above were using a
+ * stemmer/lowercaser/stopword combo) 2) The number of other fields (Lucene
+ * reads ALL fields off the disk when accessing just one document field - can
+ * cost dear!) 3) Use of compression on field storage - could be faster due to
+ * compression (less disk IO) or slower (more CPU burn) depending on the
+ * content.
+ *
+ * @param tpv
+ * @param tokenPositionsGuaranteedContiguous true if the token position
+ * numbers have no overlaps or gaps. If looking to eek out the last
+ * drops of performance, set to true. If in doubt, set to false.
+ */
+ public static TokenStream getTokenStream(TermPositionVector tpv,
+ boolean tokenPositionsGuaranteedContiguous) {
+ if (!tokenPositionsGuaranteedContiguous && tpv.getTermPositions(0) != null) {
+ return new TokenStreamFromTermPositionVector(tpv);
+ }
+
+ // an object used to iterate across an array of tokens
+ final class StoredTokenStream extends TokenStream {
+ Token tokens[];
+
+ int currentToken = 0;
+
+ CharTermAttribute termAtt;
+
+ OffsetAttribute offsetAtt;
+
+ PositionIncrementAttribute posincAtt;
+
+ StoredTokenStream(Token tokens[]) {
+ this.tokens = tokens;
+ termAtt = addAttribute(CharTermAttribute.class);
+ offsetAtt = addAttribute(OffsetAttribute.class);
+ posincAtt = addAttribute(PositionIncrementAttribute.class);
+ }
+
+ @Override
+ public boolean incrementToken() throws IOException {
+ if (currentToken >= tokens.length) {
+ return false;
+ }
+ Token token = tokens[currentToken++];
+ clearAttributes();
+ termAtt.setEmpty().append(token);
+ offsetAtt.setOffset(token.startOffset(), token.endOffset());
+ posincAtt
+ .setPositionIncrement(currentToken <= 1
+ || tokens[currentToken - 1].startOffset() > tokens[currentToken - 2]
+ .startOffset() ? 1 : 0);
+ return true;
+ }
+ }
+ // code to reconstruct the original sequence of Tokens
+ String[] terms = tpv.getTerms();
+ int[] freq = tpv.getTermFrequencies();
+ int totalTokens = 0;
+ for (int t = 0; t < freq.length; t++) {
+ totalTokens += freq[t];
+ }
+ Token tokensInOriginalOrder[] = new Token[totalTokens];
+ ArrayList<Token> unsortedTokens = null;
+ for (int t = 0; t < freq.length; t++) {
+ TermVectorOffsetInfo[] offsets = tpv.getOffsets(t);
+ if (offsets == null) {
+ throw new IllegalArgumentException(
+ "Required TermVector Offset information was not found");
+ }
+
+ int[] pos = null;
+ if (tokenPositionsGuaranteedContiguous) {
+ // try get the token position info to speed up assembly of tokens into
+ // sorted sequence
+ pos = tpv.getTermPositions(t);
+ }
+ if (pos == null) {
+ // tokens NOT stored with positions or not guaranteed contiguous - must
+ // add to list and sort later
+ if (unsortedTokens == null) {
+ unsortedTokens = new ArrayList<Token>();
+ }
+ for (int tp = 0; tp < offsets.length; tp++) {
+ Token token = new Token(terms[t], offsets[tp].getStartOffset(), offsets[tp]
+ .getEndOffset());
+ unsortedTokens.add(token);
+ }
+ } else {
+ // We have positions stored and a guarantee that the token position
+ // information is contiguous
+
+ // This may be fast BUT wont work if Tokenizers used which create >1
+ // token in same position or
+ // creates jumps in position numbers - this code would fail under those
+ // circumstances
+
+ // tokens stored with positions - can use this to index straight into
+ // sorted array
+ for (int tp = 0; tp < pos.length; tp++) {
+ Token token = new Token(terms[t], offsets[tp].getStartOffset(),
+ offsets[tp].getEndOffset());
+ tokensInOriginalOrder[pos[tp]] = token;
+ }
+ }
+ }
+ // If the field has been stored without position data we must perform a sort
+ if (unsortedTokens != null) {
+ tokensInOriginalOrder = unsortedTokens.toArray(new Token[unsortedTokens
+ .size()]);
+ ArrayUtil.mergeSort(tokensInOriginalOrder, new Comparator<Token>() {
+ public int compare(Token t1, Token t2) {
+ if (t1.startOffset() == t2.startOffset()) return t1.endOffset()
+ - t2.endOffset();
+ else return t1.startOffset() - t2.startOffset();
+ }
+ });
+ }
+ return new StoredTokenStream(tokensInOriginalOrder);
+ }
+
+ public static TokenStream getTokenStream(IndexReader reader, int docId,
+ String field) throws IOException {
+ TermFreqVector tfv = reader.getTermFreqVector(docId, field);
+ if (tfv == null) {
+ throw new IllegalArgumentException(field + " in doc #" + docId
+ + "does not have any term position data stored");
+ }
+ if (tfv instanceof TermPositionVector) {
+ TermPositionVector tpv = (TermPositionVector) reader.getTermFreqVector(
+ docId, field);
+ return getTokenStream(tpv);
+ }
+ throw new IllegalArgumentException(field + " in doc #" + docId
+ + "does not have any term position data stored");
+ }
+
+ // convenience method
+ public static TokenStream getTokenStream(IndexReader reader, int docId,
+ String field, Analyzer analyzer) throws IOException {
+ Document doc = reader.document(docId);
+ return getTokenStream(doc, field, analyzer);
+ }
+
+ public static TokenStream getTokenStream(Document doc, String field,
+ Analyzer analyzer) {
+ String contents = doc.get(field);
+ if (contents == null) {
+ throw new IllegalArgumentException("Field " + field
+ + " in document is not stored and cannot be analyzed");
+ }
+ return getTokenStream(field, contents, analyzer);
+ }
+
+ // convenience method
+ public static TokenStream getTokenStream(String field, String contents,
+ Analyzer analyzer) {
+ try {
+ return analyzer.reusableTokenStream(field, new StringReader(contents));
+ } catch (IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ }
+
+}
fnp / pylucene.git / blobdiff