--- /dev/null
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.tokenattributes.TermAttributeImpl;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.index.Payload;
+import org.apache.lucene.index.TermPositions; // for javadoc
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
+
+/**
+ A Token is an occurrence of a term from the text of a field. It consists of
+ a term's text, the start and end offset of the term in the text of the field,
+ and a type string.
+ <p>
+ The start and end offsets permit applications to re-associate a token with
+ its source text, e.g., to display highlighted query terms in a document
+ browser, or to show matching text fragments in a <abbr title="KeyWord In Context">KWIC</abbr>
+ display, etc.
+ <p>
+ The type is a string, assigned by a lexical analyzer
+ (a.k.a. tokenizer), naming the lexical or syntactic class that the token
+ belongs to. For example an end of sentence marker token might be implemented
+ with type "eos". The default token type is "word".
+ <p>
+ A Token can optionally have metadata (a.k.a. Payload) in the form of a variable
+ length byte array. Use {@link TermPositions#getPayloadLength()} and
+ {@link TermPositions#getPayload(byte[], int)} to retrieve the payloads from the index.
+
+ <br><br>
+
+ <p><b>NOTE:</b> As of 2.9, Token implements all {@link Attribute} interfaces
+ that are part of core Lucene and can be found in the {@code tokenattributes} subpackage.
+ Even though it is not necessary to use Token anymore, with the new TokenStream API it can
+ be used as convenience class that implements all {@link Attribute}s, which is especially useful
+ to easily switch from the old to the new TokenStream API.
+
+ <br><br>
+
+ <p>Tokenizers and TokenFilters should try to re-use a Token
+ instance when possible for best performance, by
+ implementing the {@link TokenStream#incrementToken()} API.
+ Failing that, to create a new Token you should first use
+ one of the constructors that starts with null text. To load
+ the token from a char[] use {@link #copyBuffer(char[], int, int)}.
+ To load from a String use {@link #setEmpty} followed by {@link #append(CharSequence)} or {@link #append(CharSequence, int, int)}.
+ Alternatively you can get the Token's termBuffer by calling either {@link #buffer()},
+ if you know that your text is shorter than the capacity of the termBuffer
+ or {@link #resizeBuffer(int)}, if there is any possibility
+ that you may need to grow the buffer. Fill in the characters of your term into this
+ buffer, with {@link String#getChars(int, int, char[], int)} if loading from a string,
+ or with {@link System#arraycopy(Object, int, Object, int, int)}, and finally call {@link #setLength(int)} to
+ set the length of the term text. See <a target="_top"
+ href="https://issues.apache.org/jira/browse/LUCENE-969">LUCENE-969</a>
+ for details.</p>
+ <p>Typical Token reuse patterns:
+ <ul>
+ <li> Copying text from a string (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre>
+ return reusableToken.reinit(string, startOffset, endOffset[, type]);
+ </pre>
+ </li>
+ <li> Copying some text from a string (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre>
+ return reusableToken.reinit(string, 0, string.length(), startOffset, endOffset[, type]);
+ </pre>
+ </li>
+ </li>
+ <li> Copying text from char[] buffer (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre>
+ return reusableToken.reinit(buffer, 0, buffer.length, startOffset, endOffset[, type]);
+ </pre>
+ </li>
+ <li> Copying some text from a char[] buffer (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre>
+ return reusableToken.reinit(buffer, start, end - start, startOffset, endOffset[, type]);
+ </pre>
+ </li>
+ <li> Copying from one one Token to another (type is reset to {@link #DEFAULT_TYPE} if not specified):<br/>
+ <pre>
+ return reusableToken.reinit(source.buffer(), 0, source.length(), source.startOffset(), source.endOffset()[, source.type()]);
+ </pre>
+ </li>
+ </ul>
+ A few things to note:
+ <ul>
+ <li>clear() initializes all of the fields to default values. This was changed in contrast to Lucene 2.4, but should affect no one.</li>
+ <li>Because <code>TokenStreams</code> can be chained, one cannot assume that the <code>Token's</code> current type is correct.</li>
+ <li>The startOffset and endOffset represent the start and offset in the source text, so be careful in adjusting them.</li>
+ <li>When caching a reusable token, clone it. When injecting a cached token into a stream that can be reset, clone it again.</li>
+ </ul>
+ </p>
+ <p>
+ <b>Please note:</b> With Lucene 3.1, the <code>{@linkplain #toString toString()}</code> method had to be changed to match the
+ {@link CharSequence} interface introduced by the interface {@link org.apache.lucene.analysis.tokenattributes.CharTermAttribute}.
+ This method now only prints the term text, no additional information anymore.
+ </p>
+ @see org.apache.lucene.index.Payload
+*/
+// TODO: change superclass to CharTermAttribute in 4.0! Maybe deprecate the whole class?
+public class Token extends TermAttributeImpl
+ implements TypeAttribute, PositionIncrementAttribute,
+ FlagsAttribute, OffsetAttribute, PayloadAttribute {
+
+ private int startOffset,endOffset;
+ private String type = DEFAULT_TYPE;
+ private int flags;
+ private Payload payload;
+ private int positionIncrement = 1;
+
+ /** Constructs a Token will null text. */
+ public Token() {
+ }
+
+ /** Constructs a Token with null text and start & end
+ * offsets.
+ * @param start start offset in the source text
+ * @param end end offset in the source text */
+ public Token(int start, int end) {
+ startOffset = start;
+ endOffset = end;
+ }
+
+ /** Constructs a Token with null text and start & end
+ * offsets plus the Token type.
+ * @param start start offset in the source text
+ * @param end end offset in the source text
+ * @param typ the lexical type of this Token */
+ public Token(int start, int end, String typ) {
+ startOffset = start;
+ endOffset = end;
+ type = typ;
+ }
+
+ /**
+ * Constructs a Token with null text and start & end
+ * offsets plus flags. NOTE: flags is EXPERIMENTAL.
+ * @param start start offset in the source text
+ * @param end end offset in the source text
+ * @param flags The bits to set for this token
+ */
+ public Token(int start, int end, int flags) {
+ startOffset = start;
+ endOffset = end;
+ this.flags = flags;
+ }
+
+ /** Constructs a Token with the given term text, and start
+ * & end offsets. The type defaults to "word."
+ * <b>NOTE:</b> for better indexing speed you should
+ * instead use the char[] termBuffer methods to set the
+ * term text.
+ * @param text term text
+ * @param start start offset
+ * @param end end offset
+ */
+ public Token(String text, int start, int end) {
+ append(text);
+ startOffset = start;
+ endOffset = end;
+ }
+
+ /** Constructs a Token with the given text, start and end
+ * offsets, & type. <b>NOTE:</b> for better indexing
+ * speed you should instead use the char[] termBuffer
+ * methods to set the term text.
+ * @param text term text
+ * @param start start offset
+ * @param end end offset
+ * @param typ token type
+ */
+ public Token(String text, int start, int end, String typ) {
+ append(text);
+ startOffset = start;
+ endOffset = end;
+ type = typ;
+ }
+
+ /**
+ * Constructs a Token with the given text, start and end
+ * offsets, & type. <b>NOTE:</b> for better indexing
+ * speed you should instead use the char[] termBuffer
+ * methods to set the term text.
+ * @param text
+ * @param start
+ * @param end
+ * @param flags token type bits
+ */
+ public Token(String text, int start, int end, int flags) {
+ append(text);
+ startOffset = start;
+ endOffset = end;
+ this.flags = flags;
+ }
+
+ /**
+ * Constructs a Token with the given term buffer (offset
+ * & length), start and end
+ * offsets
+ * @param startTermBuffer
+ * @param termBufferOffset
+ * @param termBufferLength
+ * @param start
+ * @param end
+ */
+ public Token(char[] startTermBuffer, int termBufferOffset, int termBufferLength, int start, int end) {
+ copyBuffer(startTermBuffer, termBufferOffset, termBufferLength);
+ startOffset = start;
+ endOffset = end;
+ }
+
+ /** Set the position increment. This determines the position of this token
+ * relative to the previous Token in a {@link TokenStream}, used in phrase
+ * searching.
+ *
+ * <p>The default value is one.
+ *
+ * <p>Some common uses for this are:<ul>
+ *
+ * <li>Set it to zero to put multiple terms in the same position. This is
+ * useful if, e.g., a word has multiple stems. Searches for phrases
+ * including either stem will match. In this case, all but the first stem's
+ * increment should be set to zero: the increment of the first instance
+ * should be one. Repeating a token with an increment of zero can also be
+ * used to boost the scores of matches on that token.
+ *
+ * <li>Set it to values greater than one to inhibit exact phrase matches.
+ * If, for example, one does not want phrases to match across removed stop
+ * words, then one could build a stop word filter that removes stop words and
+ * also sets the increment to the number of stop words removed before each
+ * non-stop word. Then exact phrase queries will only match when the terms
+ * occur with no intervening stop words.
+ *
+ * </ul>
+ * @param positionIncrement the distance from the prior term
+ * @see org.apache.lucene.index.TermPositions
+ */
+ public void setPositionIncrement(int positionIncrement) {
+ if (positionIncrement < 0)
+ throw new IllegalArgumentException
+ ("Increment must be zero or greater: " + positionIncrement);
+ this.positionIncrement = positionIncrement;
+ }
+
+ /** Returns the position increment of this Token.
+ * @see #setPositionIncrement
+ */
+ public int getPositionIncrement() {
+ return positionIncrement;
+ }
+
+ /** Returns this Token's starting offset, the position of the first character
+ corresponding to this token in the source text.
+
+ Note that the difference between endOffset() and startOffset() may not be
+ equal to {@link #length}, as the term text may have been altered by a
+ stemmer or some other filter. */
+ public final int startOffset() {
+ return startOffset;
+ }
+
+ /** Set the starting offset.
+ @see #startOffset() */
+ public void setStartOffset(int offset) {
+ this.startOffset = offset;
+ }
+
+ /** Returns this Token's ending offset, one greater than the position of the
+ last character corresponding to this token in the source text. The length
+ of the token in the source text is (endOffset - startOffset). */
+ public final int endOffset() {
+ return endOffset;
+ }
+
+ /** Set the ending offset.
+ @see #endOffset() */
+ public void setEndOffset(int offset) {
+ this.endOffset = offset;
+ }
+
+ /** Set the starting and ending offset.
+ @see #startOffset() and #endOffset()*/
+ public void setOffset(int startOffset, int endOffset) {
+ this.startOffset = startOffset;
+ this.endOffset = endOffset;
+ }
+
+ /** Returns this Token's lexical type. Defaults to "word". */
+ public final String type() {
+ return type;
+ }
+
+ /** Set the lexical type.
+ @see #type() */
+ public final void setType(String type) {
+ this.type = type;
+ }
+
+ /**
+ * <p/>
+ *
+ * Get the bitset for any bits that have been set. This is completely distinct from {@link #type()}, although they do share similar purposes.
+ * The flags can be used to encode information about the token for use by other {@link org.apache.lucene.analysis.TokenFilter}s.
+ *
+ *
+ * @return The bits
+ * @lucene.experimental While we think this is here to stay, we may want to change it to be a long.
+ */
+ public int getFlags() {
+ return flags;
+ }
+
+ /**
+ * @see #getFlags()
+ */
+ public void setFlags(int flags) {
+ this.flags = flags;
+ }
+
+ /**
+ * Returns this Token's payload.
+ */
+ public Payload getPayload() {
+ return this.payload;
+ }
+
+ /**
+ * Sets this Token's payload.
+ */
+ public void setPayload(Payload payload) {
+ this.payload = payload;
+ }
+
+ /** Resets the term text, payload, flags, and positionIncrement,
+ * startOffset, endOffset and token type to default.
+ */
+ @Override
+ public void clear() {
+ super.clear();
+ payload = null;
+ positionIncrement = 1;
+ flags = 0;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
+
+ @Override
+ public Object clone() {
+ Token t = (Token)super.clone();
+ // Do a deep clone
+ if (payload != null) {
+ t.payload = (Payload) payload.clone();
+ }
+ return t;
+ }
+
+ /** Makes a clone, but replaces the term buffer &
+ * start/end offset in the process. This is more
+ * efficient than doing a full clone (and then calling
+ * {@link #copyBuffer}) because it saves a wasted copy of the old
+ * termBuffer. */
+ public Token clone(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
+ final Token t = new Token(newTermBuffer, newTermOffset, newTermLength, newStartOffset, newEndOffset);
+ t.positionIncrement = positionIncrement;
+ t.flags = flags;
+ t.type = type;
+ if (payload != null)
+ t.payload = (Payload) payload.clone();
+ return t;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this)
+ return true;
+
+ if (obj instanceof Token) {
+ final Token other = (Token) obj;
+ return (startOffset == other.startOffset &&
+ endOffset == other.endOffset &&
+ flags == other.flags &&
+ positionIncrement == other.positionIncrement &&
+ (type == null ? other.type == null : type.equals(other.type)) &&
+ (payload == null ? other.payload == null : payload.equals(other.payload)) &&
+ super.equals(obj)
+ );
+ } else
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ int code = super.hashCode();
+ code = code * 31 + startOffset;
+ code = code * 31 + endOffset;
+ code = code * 31 + flags;
+ code = code * 31 + positionIncrement;
+ if (type != null)
+ code = code * 31 + type.hashCode();
+ if (payload != null)
+ code = code * 31 + payload.hashCode();
+ return code;
+ }
+
+ // like clear() but doesn't clear termBuffer/text
+ private void clearNoTermBuffer() {
+ payload = null;
+ positionIncrement = 1;
+ flags = 0;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #copyBuffer(char[], int, int)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset},
+ * {@link #setType}
+ * @return this Token instance */
+ public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) {
+ clearNoTermBuffer();
+ copyBuffer(newTermBuffer, newTermOffset, newTermLength);
+ payload = null;
+ positionIncrement = 1;
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #copyBuffer(char[], int, int)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType} on Token.DEFAULT_TYPE
+ * @return this Token instance */
+ public Token reinit(char[] newTermBuffer, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
+ clearNoTermBuffer();
+ copyBuffer(newTermBuffer, newTermOffset, newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType}
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newStartOffset, int newEndOffset, String newType) {
+ clear();
+ append(newTerm);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence, int, int)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType}
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset, String newType) {
+ clear();
+ append(newTerm, newTermOffset, newTermOffset + newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = newType;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType} on Token.DEFAULT_TYPE
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newStartOffset, int newEndOffset) {
+ clear();
+ append(newTerm);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /** Shorthand for calling {@link #clear},
+ * {@link #append(CharSequence, int, int)},
+ * {@link #setStartOffset},
+ * {@link #setEndOffset}
+ * {@link #setType} on Token.DEFAULT_TYPE
+ * @return this Token instance */
+ public Token reinit(String newTerm, int newTermOffset, int newTermLength, int newStartOffset, int newEndOffset) {
+ clear();
+ append(newTerm, newTermOffset, newTermOffset + newTermLength);
+ startOffset = newStartOffset;
+ endOffset = newEndOffset;
+ type = DEFAULT_TYPE;
+ return this;
+ }
+
+ /**
+ * Copy the prototype token's fields into this one. Note: Payloads are shared.
+ * @param prototype
+ */
+ public void reinit(Token prototype) {
+ copyBuffer(prototype.buffer(), 0, prototype.length());
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ /**
+ * Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.
+ * @param prototype
+ * @param newTerm
+ */
+ public void reinit(Token prototype, String newTerm) {
+ setEmpty().append(newTerm);
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ /**
+ * Copy the prototype token's fields into this one, with a different term. Note: Payloads are shared.
+ * @param prototype
+ * @param newTermBuffer
+ * @param offset
+ * @param length
+ */
+ public void reinit(Token prototype, char[] newTermBuffer, int offset, int length) {
+ copyBuffer(newTermBuffer, offset, length);
+ positionIncrement = prototype.positionIncrement;
+ flags = prototype.flags;
+ startOffset = prototype.startOffset;
+ endOffset = prototype.endOffset;
+ type = prototype.type;
+ payload = prototype.payload;
+ }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ if (target instanceof Token) {
+ final Token to = (Token) target;
+ to.reinit(this);
+ // reinit shares the payload, so clone it:
+ if (payload !=null) {
+ to.payload = (Payload) payload.clone();
+ }
+ } else {
+ super.copyTo(target);
+ ((OffsetAttribute) target).setOffset(startOffset, endOffset);
+ ((PositionIncrementAttribute) target).setPositionIncrement(positionIncrement);
+ ((PayloadAttribute) target).setPayload((payload == null) ? null : (Payload) payload.clone());
+ ((FlagsAttribute) target).setFlags(flags);
+ ((TypeAttribute) target).setType(type);
+ }
+ }
+
+ @Override
+ public void reflectWith(AttributeReflector reflector) {
+ super.reflectWith(reflector);
+ reflector.reflect(OffsetAttribute.class, "startOffset", startOffset);
+ reflector.reflect(OffsetAttribute.class, "endOffset", endOffset);
+ reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", positionIncrement);
+ reflector.reflect(PayloadAttribute.class, "payload", payload);
+ reflector.reflect(FlagsAttribute.class, "flags", flags);
+ reflector.reflect(TypeAttribute.class, "type", type);
+ }
+
+ /** Convenience factory that returns <code>Token</code> as implementation for the basic
+ * attributes and return the default impl (with "Impl" appended) for all other
+ * attributes.
+ * @since 3.0
+ */
+ public static final AttributeSource.AttributeFactory TOKEN_ATTRIBUTE_FACTORY =
+ new TokenAttributeFactory(AttributeSource.AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
+
+ /** <b>Expert:</b> Creates a TokenAttributeFactory returning {@link Token} as instance for the basic attributes
+ * and for all other attributes calls the given delegate factory.
+ * @since 3.0
+ */
+ public static final class TokenAttributeFactory extends AttributeSource.AttributeFactory {
+
+ private final AttributeSource.AttributeFactory delegate;
+
+ /** <b>Expert</b>: Creates an AttributeFactory returning {@link Token} as instance for the basic attributes
+ * and for all other attributes calls the given delegate factory. */
+ public TokenAttributeFactory(AttributeSource.AttributeFactory delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public AttributeImpl createAttributeInstance(Class<? extends Attribute> attClass) {
+ return attClass.isAssignableFrom(Token.class)
+ ? new Token() : delegate.createAttributeInstance(attClass);
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) return true;
+ if (other instanceof TokenAttributeFactory) {
+ final TokenAttributeFactory af = (TokenAttributeFactory) other;
+ return this.delegate.equals(af.delegate);
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return delegate.hashCode() ^ 0x0a45aa31;
+ }
+ }
+
+}
fnp / pylucene.git / blobdiff