1 package org.apache.lucene.index;
4 * Copyright 2004 The Apache Software Foundation
6 * Licensed under the Apache License, Version 2.0 (the "License"); you may not
7 * use this file except in compliance with the License. You may obtain a copy of
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing, software
13 * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14 * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
15 * License for the specific language governing permissions and limitations under
19 import java.io.IOException;
20 import java.util.List;
22 import org.apache.lucene.document.Document;
23 import org.apache.lucene.document.Fieldable;
24 import org.apache.lucene.document.NumericField;
25 import org.apache.lucene.store.Directory;
26 import org.apache.lucene.store.IndexInput;
27 import org.apache.lucene.store.IndexOutput;
28 import org.apache.lucene.store.RAMOutputStream;
29 import org.apache.lucene.util.IOUtils;
31 final class FieldsWriter {
32 static final int FIELD_IS_TOKENIZED = 1 << 0;
33 static final int FIELD_IS_BINARY = 1 << 1;
35 /** @deprecated Kept for backwards-compatibility with <3.0 indexes; will be removed in 4.0 */
37 static final int FIELD_IS_COMPRESSED = 1 << 2;
39 private static final int _NUMERIC_BIT_SHIFT = 3;
40 static final int FIELD_IS_NUMERIC_MASK = 0x07 << _NUMERIC_BIT_SHIFT;
42 static final int FIELD_IS_NUMERIC_INT = 1 << _NUMERIC_BIT_SHIFT;
43 static final int FIELD_IS_NUMERIC_LONG = 2 << _NUMERIC_BIT_SHIFT;
44 static final int FIELD_IS_NUMERIC_FLOAT = 3 << _NUMERIC_BIT_SHIFT;
45 static final int FIELD_IS_NUMERIC_DOUBLE = 4 << _NUMERIC_BIT_SHIFT;
46 // currently unused: static final int FIELD_IS_NUMERIC_SHORT = 5 << _NUMERIC_BIT_SHIFT;
47 // currently unused: static final int FIELD_IS_NUMERIC_BYTE = 6 << _NUMERIC_BIT_SHIFT;
49 // the next possible bits are: 1 << 6; 1 << 7
52 static final int FORMAT = 0;
54 // Changed strings to UTF8
55 static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = 1;
57 // Lucene 3.0: Removal of compressed fields
58 static final int FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS = 2;
60 // Lucene 3.2: NumericFields are stored in binary format
61 static final int FORMAT_LUCENE_3_2_NUMERIC_FIELDS = 3;
63 // NOTE: if you introduce a new format, make it 1 higher
64 // than the current one, and always change this if you
65 // switch to a new format!
66 static final int FORMAT_CURRENT = FORMAT_LUCENE_3_2_NUMERIC_FIELDS;
68 private FieldInfos fieldInfos;
70 // If null - we were supplied with streams, if notnull - we manage them ourselves
71 private Directory directory;
72 private String segment;
73 private IndexOutput fieldsStream;
74 private IndexOutput indexStream;
76 FieldsWriter(Directory directory, String segment, FieldInfos fn) throws IOException {
77 this.directory = directory;
78 this.segment = segment;
81 boolean success = false;
83 fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_EXTENSION));
84 indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_INDEX_EXTENSION));
86 fieldsStream.writeInt(FORMAT_CURRENT);
87 indexStream.writeInt(FORMAT_CURRENT);
97 FieldsWriter(IndexOutput fdx, IndexOutput fdt, FieldInfos fn) {
105 void setFieldsStream(IndexOutput stream) {
106 this.fieldsStream = stream;
109 // Writes the contents of buffer into the fields stream
110 // and adds a new entry for this document into the index
111 // stream. This assumes the buffer was already written
112 // in the correct fields format.
113 void flushDocument(int numStoredFields, RAMOutputStream buffer) throws IOException {
114 indexStream.writeLong(fieldsStream.getFilePointer());
115 fieldsStream.writeVInt(numStoredFields);
116 buffer.writeTo(fieldsStream);
119 void skipDocument() throws IOException {
120 indexStream.writeLong(fieldsStream.getFilePointer());
121 fieldsStream.writeVInt(0);
124 void close() throws IOException {
125 if (directory != null) {
127 IOUtils.close(fieldsStream, indexStream);
129 fieldsStream = indexStream = null;
135 if (directory != null) {
138 } catch (IOException ignored) {
141 directory.deleteFile(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_EXTENSION));
142 } catch (IOException ignored) {
145 directory.deleteFile(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_INDEX_EXTENSION));
146 } catch (IOException ignored) {
151 final void writeField(FieldInfo fi, Fieldable field) throws IOException {
152 fieldsStream.writeVInt(fi.number);
154 if (field.isTokenized())
155 bits |= FIELD_IS_TOKENIZED;
156 if (field.isBinary())
157 bits |= FIELD_IS_BINARY;
158 if (field instanceof NumericField) {
159 switch (((NumericField) field).getDataType()) {
161 bits |= FIELD_IS_NUMERIC_INT; break;
163 bits |= FIELD_IS_NUMERIC_LONG; break;
165 bits |= FIELD_IS_NUMERIC_FLOAT; break;
167 bits |= FIELD_IS_NUMERIC_DOUBLE; break;
169 assert false : "Should never get here";
172 fieldsStream.writeByte((byte) bits);
174 if (field.isBinary()) {
178 data = field.getBinaryValue();
179 len = field.getBinaryLength();
180 offset = field.getBinaryOffset();
182 fieldsStream.writeVInt(len);
183 fieldsStream.writeBytes(data, offset, len);
184 } else if (field instanceof NumericField) {
185 final NumericField nf = (NumericField) field;
186 final Number n = nf.getNumericValue();
187 switch (nf.getDataType()) {
189 fieldsStream.writeInt(n.intValue()); break;
191 fieldsStream.writeLong(n.longValue()); break;
193 fieldsStream.writeInt(Float.floatToIntBits(n.floatValue())); break;
195 fieldsStream.writeLong(Double.doubleToLongBits(n.doubleValue())); break;
197 assert false : "Should never get here";
200 fieldsStream.writeString(field.stringValue());
204 /** Bulk write a contiguous series of documents. The
205 * lengths array is the length (in bytes) of each raw
206 * document. The stream IndexInput is the
207 * fieldsStream from which we should bulk-copy all
209 final void addRawDocuments(IndexInput stream, int[] lengths, int numDocs) throws IOException {
210 long position = fieldsStream.getFilePointer();
211 long start = position;
212 for(int i=0;i<numDocs;i++) {
213 indexStream.writeLong(position);
214 position += lengths[i];
216 fieldsStream.copyBytes(stream, position-start);
217 assert fieldsStream.getFilePointer() == position;
220 final void addDocument(Document doc) throws IOException {
221 indexStream.writeLong(fieldsStream.getFilePointer());
224 List<Fieldable> fields = doc.getFields();
225 for (Fieldable field : fields) {
226 if (field.isStored())
229 fieldsStream.writeVInt(storedCount);
233 for (Fieldable field : fields) {
234 if (field.isStored())
235 writeField(fieldInfos.fieldInfo(field.name()), field);