1 package org.apache.lucene.analysis;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import org.apache.lucene.util.AttributeSource;
21 import org.apache.lucene.util.NumericUtils;
22 import org.apache.lucene.document.NumericField; // for javadocs
23 import org.apache.lucene.search.NumericRangeQuery; // for javadocs
24 import org.apache.lucene.search.NumericRangeFilter; // for javadocs
25 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
26 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
27 import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
30 * <b>Expert:</b> This class provides a {@link TokenStream}
31 * for indexing numeric values that can be used by {@link
32 * NumericRangeQuery} or {@link NumericRangeFilter}.
34 * <p>Note that for simple usage, {@link NumericField} is
35 * recommended. {@link NumericField} disables norms and
36 * term freqs, as they are not usually needed during
37 * searching. If you need to change these settings, you
38 * should use this class.
40 * <p>See {@link NumericField} for capabilities of fields
41 * indexed numerically.</p>
43 * <p>Here's an example usage, for an <code>int</code> field:
46 * Field field = new Field(name, new NumericTokenStream(precisionStep).setIntValue(value));
47 * field.setOmitNorms(true);
48 * field.setIndexOptions(IndexOptions.DOCS_ONLY);
49 * document.add(field);
52 * <p>For optimal performance, re-use the TokenStream and Field instance
53 * for more than one document:
56 * NumericTokenStream stream = new NumericTokenStream(precisionStep);
57 * Field field = new Field(name, stream);
58 * field.setOmitNorms(true);
59 * field.setIndexOptions(IndexOptions.DOCS_ONLY);
60 * Document document = new Document();
61 * document.add(field);
63 * for(all documents) {
64 * stream.setIntValue(value)
65 * writer.addDocument(document);
69 * <p>This stream is not intended to be used in analyzers;
70 * it's more for iterating the different precisions during
71 * indexing a specific numeric value.</p>
73 * <p><b>NOTE</b>: as token streams are only consumed once
74 * the document is added to the index, if you index more
75 * than one numeric field, use a separate <code>NumericTokenStream</code>
76 * instance for each.</p>
78 * <p>See {@link NumericRangeQuery} for more details on the
80 * href="../search/NumericRangeQuery.html#precisionStepDesc"><code>precisionStep</code></a>
81 * parameter as well as how numeric fields work under the hood.</p>
85 public final class NumericTokenStream extends TokenStream {
87 /** The full precision token gets this token type assigned. */
88 public static final String TOKEN_TYPE_FULL_PREC = "fullPrecNumeric";
90 /** The lower precision tokens gets this token type assigned. */
91 public static final String TOKEN_TYPE_LOWER_PREC = "lowerPrecNumeric";
94 * Creates a token stream for numeric values using the default <code>precisionStep</code>
95 * {@link NumericUtils#PRECISION_STEP_DEFAULT} (4). The stream is not yet initialized,
96 * before using set a value using the various set<em>???</em>Value() methods.
98 public NumericTokenStream() {
99 this(NumericUtils.PRECISION_STEP_DEFAULT);
103 * Creates a token stream for numeric values with the specified
104 * <code>precisionStep</code>. The stream is not yet initialized,
105 * before using set a value using the various set<em>???</em>Value() methods.
107 public NumericTokenStream(final int precisionStep) {
109 this.precisionStep = precisionStep;
110 if (precisionStep < 1)
111 throw new IllegalArgumentException("precisionStep must be >=1");
115 * Expert: Creates a token stream for numeric values with the specified
116 * <code>precisionStep</code> using the given {@link AttributeSource}.
117 * The stream is not yet initialized,
118 * before using set a value using the various set<em>???</em>Value() methods.
120 public NumericTokenStream(AttributeSource source, final int precisionStep) {
122 this.precisionStep = precisionStep;
123 if (precisionStep < 1)
124 throw new IllegalArgumentException("precisionStep must be >=1");
128 * Expert: Creates a token stream for numeric values with the specified
129 * <code>precisionStep</code> using the given
130 * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
131 * The stream is not yet initialized,
132 * before using set a value using the various set<em>???</em>Value() methods.
134 public NumericTokenStream(AttributeFactory factory, final int precisionStep) {
136 this.precisionStep = precisionStep;
137 if (precisionStep < 1)
138 throw new IllegalArgumentException("precisionStep must be >=1");
142 * Initializes the token stream with the supplied <code>long</code> value.
143 * @param value the value, for which this TokenStream should enumerate tokens.
144 * @return this instance, because of this you can use it the following way:
145 * <code>new Field(name, new NumericTokenStream(precisionStep).setLongValue(value))</code>
147 public NumericTokenStream setLongValue(final long value) {
155 * Initializes the token stream with the supplied <code>int</code> value.
156 * @param value the value, for which this TokenStream should enumerate tokens.
157 * @return this instance, because of this you can use it the following way:
158 * <code>new Field(name, new NumericTokenStream(precisionStep).setIntValue(value))</code>
160 public NumericTokenStream setIntValue(final int value) {
168 * Initializes the token stream with the supplied <code>double</code> value.
169 * @param value the value, for which this TokenStream should enumerate tokens.
170 * @return this instance, because of this you can use it the following way:
171 * <code>new Field(name, new NumericTokenStream(precisionStep).setDoubleValue(value))</code>
173 public NumericTokenStream setDoubleValue(final double value) {
174 this.value = NumericUtils.doubleToSortableLong(value);
181 * Initializes the token stream with the supplied <code>float</code> value.
182 * @param value the value, for which this TokenStream should enumerate tokens.
183 * @return this instance, because of this you can use it the following way:
184 * <code>new Field(name, new NumericTokenStream(precisionStep).setFloatValue(value))</code>
186 public NumericTokenStream setFloatValue(final float value) {
187 this.value = NumericUtils.floatToSortableInt(value);
194 public void reset() {
196 throw new IllegalStateException("call set???Value() before usage");
201 public boolean incrementToken() {
203 throw new IllegalStateException("call set???Value() before usage");
204 if (shift >= valSize)
211 buffer = termAtt.resizeBuffer(NumericUtils.BUF_SIZE_LONG);
212 termAtt.setLength(NumericUtils.longToPrefixCoded(value, shift, buffer));
216 buffer = termAtt.resizeBuffer(NumericUtils.BUF_SIZE_INT);
217 termAtt.setLength(NumericUtils.intToPrefixCoded((int) value, shift, buffer));
222 throw new IllegalArgumentException("valSize must be 32 or 64");
225 typeAtt.setType((shift == 0) ? TOKEN_TYPE_FULL_PREC : TOKEN_TYPE_LOWER_PREC);
226 posIncrAtt.setPositionIncrement((shift == 0) ? 1 : 0);
227 shift += precisionStep;
232 public String toString() {
233 final StringBuilder sb = new StringBuilder("(numeric,valSize=").append(valSize);
234 sb.append(",precisionStep=").append(precisionStep).append(')');
235 return sb.toString();
238 /** Returns the precision step. */
239 public int getPrecisionStep() {
240 return precisionStep;
244 private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
245 private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
246 private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
248 private int shift = 0, valSize = 0; // valSize==0 means not initialized
249 private final int precisionStep;
251 private long value = 0L;