--- /dev/null
+package org.apache.lucene.search.highlight;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Iterator;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.util.PriorityQueue;
+
+/**
+ * Class used to markup highlighted terms found in the best sections of a
+ * text, using configurable {@link Fragmenter}, {@link Scorer}, {@link Formatter},
+ * {@link Encoder} and tokenizers.
+ */
+public class Highlighter
+{
+ public static final int DEFAULT_MAX_CHARS_TO_ANALYZE = 50*1024;
+
+ private int maxDocCharsToAnalyze = DEFAULT_MAX_CHARS_TO_ANALYZE;
+ private Formatter formatter;
+ private Encoder encoder;
+ private Fragmenter textFragmenter=new SimpleFragmenter();
+ private Scorer fragmentScorer=null;
+
+ public Highlighter(Scorer fragmentScorer)
+ {
+ this(new SimpleHTMLFormatter(),fragmentScorer);
+ }
+
+
+ public Highlighter(Formatter formatter, Scorer fragmentScorer)
+ {
+ this(formatter,new DefaultEncoder(),fragmentScorer);
+ }
+
+
+ public Highlighter(Formatter formatter, Encoder encoder, Scorer fragmentScorer)
+ {
+ this.formatter = formatter;
+ this.encoder = encoder;
+ this.fragmentScorer = fragmentScorer;
+ }
+
+ /**
+ * Highlights chosen terms in a text, extracting the most relevant section.
+ * This is a convenience method that calls
+ * {@link #getBestFragment(TokenStream, String)}
+ *
+ * @param analyzer the analyzer that will be used to split <code>text</code>
+ * into chunks
+ * @param text text to highlight terms in
+ * @param fieldName Name of field used to influence analyzer's tokenization policy
+ *
+ * @return highlighted text fragment or null if no terms found
+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length
+ */
+ public final String getBestFragment(Analyzer analyzer, String fieldName,String text)
+ throws IOException, InvalidTokenOffsetsException
+ {
+ TokenStream tokenStream = analyzer.reusableTokenStream(fieldName, new StringReader(text));
+ return getBestFragment(tokenStream, text);
+ }
+
+ /**
+ * Highlights chosen terms in a text, extracting the most relevant section.
+ * The document text is analysed in chunks to record hit statistics
+ * across the document. After accumulating stats, the fragment with the highest score
+ * is returned
+ *
+ * @param tokenStream a stream of tokens identified in the text parameter, including offset information.
+ * This is typically produced by an analyzer re-parsing a document's
+ * text. Some work may be done on retrieving TokenStreams more efficiently
+ * by adding support for storing original text position data in the Lucene
+ * index but this support is not currently available (as of Lucene 1.4 rc2).
+ * @param text text to highlight terms in
+ *
+ * @return highlighted text fragment or null if no terms found
+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length
+ */
+ public final String getBestFragment(TokenStream tokenStream, String text)
+ throws IOException, InvalidTokenOffsetsException
+ {
+ String[] results = getBestFragments(tokenStream,text, 1);
+ if (results.length > 0)
+ {
+ return results[0];
+ }
+ return null;
+ }
+
+ /**
+ * Highlights chosen terms in a text, extracting the most relevant sections.
+ * This is a convenience method that calls
+ * {@link #getBestFragments(TokenStream, String, int)}
+ *
+ * @param analyzer the analyzer that will be used to split <code>text</code>
+ * into chunks
+ * @param fieldName the name of the field being highlighted (used by analyzer)
+ * @param text text to highlight terms in
+ * @param maxNumFragments the maximum number of fragments.
+ *
+ * @return highlighted text fragments (between 0 and maxNumFragments number of fragments)
+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length
+ */
+ public final String[] getBestFragments(
+ Analyzer analyzer,
+ String fieldName,
+ String text,
+ int maxNumFragments)
+ throws IOException, InvalidTokenOffsetsException
+ {
+ TokenStream tokenStream = analyzer.reusableTokenStream(fieldName, new StringReader(text));
+ return getBestFragments(tokenStream, text, maxNumFragments);
+ }
+
+ /**
+ * Highlights chosen terms in a text, extracting the most relevant sections.
+ * The document text is analysed in chunks to record hit statistics
+ * across the document. After accumulating stats, the fragments with the highest scores
+ * are returned as an array of strings in order of score (contiguous fragments are merged into
+ * one in their original order to improve readability)
+ *
+ * @param text text to highlight terms in
+ * @param maxNumFragments the maximum number of fragments.
+ *
+ * @return highlighted text fragments (between 0 and maxNumFragments number of fragments)
+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length
+ */
+ public final String[] getBestFragments(
+ TokenStream tokenStream,
+ String text,
+ int maxNumFragments)
+ throws IOException, InvalidTokenOffsetsException
+ {
+ maxNumFragments = Math.max(1, maxNumFragments); //sanity check
+
+ TextFragment[] frag =getBestTextFragments(tokenStream,text, true,maxNumFragments);
+
+ //Get text
+ ArrayList<String> fragTexts = new ArrayList<String>();
+ for (int i = 0; i < frag.length; i++)
+ {
+ if ((frag[i] != null) && (frag[i].getScore() > 0))
+ {
+ fragTexts.add(frag[i].toString());
+ }
+ }
+ return fragTexts.toArray(new String[0]);
+ }
+
+
+ /**
+ * Low level api to get the most relevant (formatted) sections of the document.
+ * This method has been made public to allow visibility of score information held in TextFragment objects.
+ * Thanks to Jason Calabrese for help in redefining the interface.
+ * @param tokenStream
+ * @param text
+ * @param maxNumFragments
+ * @param mergeContiguousFragments
+ * @throws IOException
+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length
+ */
+ public final TextFragment[] getBestTextFragments(
+ TokenStream tokenStream,
+ String text,
+ boolean mergeContiguousFragments,
+ int maxNumFragments)
+ throws IOException, InvalidTokenOffsetsException
+ {
+ ArrayList<TextFragment> docFrags = new ArrayList<TextFragment>();
+ StringBuilder newText=new StringBuilder();
+
+ CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class);
+ OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
+ tokenStream.addAttribute(PositionIncrementAttribute.class);
+ tokenStream.reset();
+
+ TextFragment currentFrag = new TextFragment(newText,newText.length(), docFrags.size());
+
+ if (fragmentScorer instanceof QueryScorer) {
+ ((QueryScorer) fragmentScorer).setMaxDocCharsToAnalyze(maxDocCharsToAnalyze);
+ }
+
+ TokenStream newStream = fragmentScorer.init(tokenStream);
+ if(newStream != null) {
+ tokenStream = newStream;
+ }
+ fragmentScorer.startFragment(currentFrag);
+ docFrags.add(currentFrag);
+
+ FragmentQueue fragQueue = new FragmentQueue(maxNumFragments);
+
+ try
+ {
+
+ String tokenText;
+ int startOffset;
+ int endOffset;
+ int lastEndOffset = 0;
+ textFragmenter.start(text, tokenStream);
+
+ TokenGroup tokenGroup=new TokenGroup(tokenStream);
+
+ for (boolean next = tokenStream.incrementToken(); next && (offsetAtt.startOffset()< maxDocCharsToAnalyze);
+ next = tokenStream.incrementToken())
+ {
+ if( (offsetAtt.endOffset()>text.length())
+ ||
+ (offsetAtt.startOffset()>text.length())
+ )
+ {
+ throw new InvalidTokenOffsetsException("Token "+ termAtt.toString()
+ +" exceeds length of provided text sized "+text.length());
+ }
+ if((tokenGroup.numTokens>0)&&(tokenGroup.isDistinct()))
+ {
+ //the current token is distinct from previous tokens -
+ // markup the cached token group info
+ startOffset = tokenGroup.matchStartOffset;
+ endOffset = tokenGroup.matchEndOffset;
+ tokenText = text.substring(startOffset, endOffset);
+ String markedUpText=formatter.highlightTerm(encoder.encodeText(tokenText), tokenGroup);
+ //store any whitespace etc from between this and last group
+ if (startOffset > lastEndOffset)
+ newText.append(encoder.encodeText(text.substring(lastEndOffset, startOffset)));
+ newText.append(markedUpText);
+ lastEndOffset=Math.max(endOffset, lastEndOffset);
+ tokenGroup.clear();
+
+ //check if current token marks the start of a new fragment
+ if(textFragmenter.isNewFragment())
+ {
+ currentFrag.setScore(fragmentScorer.getFragmentScore());
+ //record stats for a new fragment
+ currentFrag.textEndPos = newText.length();
+ currentFrag =new TextFragment(newText, newText.length(), docFrags.size());
+ fragmentScorer.startFragment(currentFrag);
+ docFrags.add(currentFrag);
+ }
+ }
+
+ tokenGroup.addToken(fragmentScorer.getTokenScore());
+
+// if(lastEndOffset>maxDocBytesToAnalyze)
+// {
+// break;
+// }
+ }
+ currentFrag.setScore(fragmentScorer.getFragmentScore());
+
+ if(tokenGroup.numTokens>0)
+ {
+ //flush the accumulated text (same code as in above loop)
+ startOffset = tokenGroup.matchStartOffset;
+ endOffset = tokenGroup.matchEndOffset;
+ tokenText = text.substring(startOffset, endOffset);
+ String markedUpText=formatter.highlightTerm(encoder.encodeText(tokenText), tokenGroup);
+ //store any whitespace etc from between this and last group
+ if (startOffset > lastEndOffset)
+ newText.append(encoder.encodeText(text.substring(lastEndOffset, startOffset)));
+ newText.append(markedUpText);
+ lastEndOffset=Math.max(lastEndOffset,endOffset);
+ }
+
+ //Test what remains of the original text beyond the point where we stopped analyzing
+ if (
+// if there is text beyond the last token considered..
+ (lastEndOffset < text.length())
+ &&
+// and that text is not too large...
+ (text.length()<= maxDocCharsToAnalyze)
+ )
+ {
+ //append it to the last fragment
+ newText.append(encoder.encodeText(text.substring(lastEndOffset)));
+ }
+
+ currentFrag.textEndPos = newText.length();
+
+ //sort the most relevant sections of the text
+ for (Iterator<TextFragment> i = docFrags.iterator(); i.hasNext();)
+ {
+ currentFrag = i.next();
+
+ //If you are running with a version of Lucene before 11th Sept 03
+ // you do not have PriorityQueue.insert() - so uncomment the code below
+ /*
+ if (currentFrag.getScore() >= minScore)
+ {
+ fragQueue.put(currentFrag);
+ if (fragQueue.size() > maxNumFragments)
+ { // if hit queue overfull
+ fragQueue.pop(); // remove lowest in hit queue
+ minScore = ((TextFragment) fragQueue.top()).getScore(); // reset minScore
+ }
+
+
+ }
+ */
+ //The above code caused a problem as a result of Christoph Goller's 11th Sept 03
+ //fix to PriorityQueue. The correct method to use here is the new "insert" method
+ // USE ABOVE CODE IF THIS DOES NOT COMPILE!
+ fragQueue.insertWithOverflow(currentFrag);
+ }
+
+ //return the most relevant fragments
+ TextFragment frag[] = new TextFragment[fragQueue.size()];
+ for (int i = frag.length - 1; i >= 0; i--)
+ {
+ frag[i] = fragQueue.pop();
+ }
+
+ //merge any contiguous fragments to improve readability
+ if(mergeContiguousFragments)
+ {
+ mergeContiguousFragments(frag);
+ ArrayList<TextFragment> fragTexts = new ArrayList<TextFragment>();
+ for (int i = 0; i < frag.length; i++)
+ {
+ if ((frag[i] != null) && (frag[i].getScore() > 0))
+ {
+ fragTexts.add(frag[i]);
+ }
+ }
+ frag= fragTexts.toArray(new TextFragment[0]);
+ }
+
+ return frag;
+
+ }
+ finally
+ {
+ if (tokenStream != null)
+ {
+ try
+ {
+ tokenStream.end();
+ tokenStream.close();
+ }
+ catch (Exception e)
+ {
+ }
+ }
+ }
+ }
+
+
+ /** Improves readability of a score-sorted list of TextFragments by merging any fragments
+ * that were contiguous in the original text into one larger fragment with the correct order.
+ * This will leave a "null" in the array entry for the lesser scored fragment.
+ *
+ * @param frag An array of document fragments in descending score
+ */
+ private void mergeContiguousFragments(TextFragment[] frag)
+ {
+ boolean mergingStillBeingDone;
+ if (frag.length > 1)
+ do
+ {
+ mergingStillBeingDone = false; //initialise loop control flag
+ //for each fragment, scan other frags looking for contiguous blocks
+ for (int i = 0; i < frag.length; i++)
+ {
+ if (frag[i] == null)
+ {
+ continue;
+ }
+ //merge any contiguous blocks
+ for (int x = 0; x < frag.length; x++)
+ {
+ if (frag[x] == null)
+ {
+ continue;
+ }
+ if (frag[i] == null)
+ {
+ break;
+ }
+ TextFragment frag1 = null;
+ TextFragment frag2 = null;
+ int frag1Num = 0;
+ int frag2Num = 0;
+ int bestScoringFragNum;
+ int worstScoringFragNum;
+ //if blocks are contiguous....
+ if (frag[i].follows(frag[x]))
+ {
+ frag1 = frag[x];
+ frag1Num = x;
+ frag2 = frag[i];
+ frag2Num = i;
+ }
+ else
+ if (frag[x].follows(frag[i]))
+ {
+ frag1 = frag[i];
+ frag1Num = i;
+ frag2 = frag[x];
+ frag2Num = x;
+ }
+ //merging required..
+ if (frag1 != null)
+ {
+ if (frag1.getScore() > frag2.getScore())
+ {
+ bestScoringFragNum = frag1Num;
+ worstScoringFragNum = frag2Num;
+ }
+ else
+ {
+ bestScoringFragNum = frag2Num;
+ worstScoringFragNum = frag1Num;
+ }
+ frag1.merge(frag2);
+ frag[worstScoringFragNum] = null;
+ mergingStillBeingDone = true;
+ frag[bestScoringFragNum] = frag1;
+ }
+ }
+ }
+ }
+ while (mergingStillBeingDone);
+ }
+
+
+ /**
+ * Highlights terms in the text , extracting the most relevant sections
+ * and concatenating the chosen fragments with a separator (typically "...").
+ * The document text is analysed in chunks to record hit statistics
+ * across the document. After accumulating stats, the fragments with the highest scores
+ * are returned in order as "separator" delimited strings.
+ *
+ * @param text text to highlight terms in
+ * @param maxNumFragments the maximum number of fragments.
+ * @param separator the separator used to intersperse the document fragments (typically "...")
+ *
+ * @return highlighted text
+ * @throws InvalidTokenOffsetsException thrown if any token's endOffset exceeds the provided text's length
+ */
+ public final String getBestFragments(
+ TokenStream tokenStream,
+ String text,
+ int maxNumFragments,
+ String separator)
+ throws IOException, InvalidTokenOffsetsException
+ {
+ String sections[] = getBestFragments(tokenStream,text, maxNumFragments);
+ StringBuilder result = new StringBuilder();
+ for (int i = 0; i < sections.length; i++)
+ {
+ if (i > 0)
+ {
+ result.append(separator);
+ }
+ result.append(sections[i]);
+ }
+ return result.toString();
+ }
+
+ public int getMaxDocCharsToAnalyze() {
+ return maxDocCharsToAnalyze;
+ }
+
+ public void setMaxDocCharsToAnalyze(int maxDocCharsToAnalyze) {
+ this.maxDocCharsToAnalyze = maxDocCharsToAnalyze;
+ }
+
+
+ public Fragmenter getTextFragmenter()
+ {
+ return textFragmenter;
+ }
+
+ /**
+ * @param fragmenter
+ */
+ public void setTextFragmenter(Fragmenter fragmenter)
+ {
+ textFragmenter = fragmenter;
+ }
+
+ /**
+ * @return Object used to score each text fragment
+ */
+ public Scorer getFragmentScorer()
+ {
+ return fragmentScorer;
+ }
+
+
+ /**
+ * @param scorer
+ */
+ public void setFragmentScorer(Scorer scorer)
+ {
+ fragmentScorer = scorer;
+ }
+
+ public Encoder getEncoder()
+ {
+ return encoder;
+ }
+ public void setEncoder(Encoder encoder)
+ {
+ this.encoder = encoder;
+ }
+}
+class FragmentQueue extends PriorityQueue<TextFragment>
+{
+ public FragmentQueue(int size)
+ {
+ initialize(size);
+ }
+
+ @Override
+ public final boolean lessThan(TextFragment fragA, TextFragment fragB)
+ {
+ if (fragA.getScore() == fragB.getScore())
+ return fragA.fragNum > fragB.fragNum;
+ else
+ return fragA.getScore() < fragB.getScore();
+ }
+}
fnp / pylucene.git / blobdiff