1 package org.apache.lucene.search;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.io.PrintStream;
22 import java.util.ArrayList;
23 import java.util.Arrays;
24 import java.util.HashMap;
25 import java.util.List;
27 import java.util.WeakHashMap;
29 import org.apache.lucene.index.IndexReader;
30 import org.apache.lucene.index.Term;
31 import org.apache.lucene.index.TermDocs;
32 import org.apache.lucene.index.TermEnum;
33 import org.apache.lucene.util.BitVector;
34 import org.apache.lucene.util.DocIdBitSet;
35 import org.apache.lucene.util.OpenBitSet;
36 import org.apache.lucene.util.StringHelper;
37 import org.apache.lucene.util.FieldCacheSanityChecker;
40 * Expert: The default cache implementation, storing all values in memory.
41 * A WeakHashMap is used for storage.
43 * <p>Created: May 19, 2004 4:40:36 PM
47 class FieldCacheImpl implements FieldCache {
49 private Map<Class<?>,Cache> caches;
53 private synchronized void init() {
54 caches = new HashMap<Class<?>,Cache>(9);
55 caches.put(Byte.TYPE, new ByteCache(this));
56 caches.put(Short.TYPE, new ShortCache(this));
57 caches.put(Integer.TYPE, new IntCache(this));
58 caches.put(Float.TYPE, new FloatCache(this));
59 caches.put(Long.TYPE, new LongCache(this));
60 caches.put(Double.TYPE, new DoubleCache(this));
61 caches.put(String.class, new StringCache(this));
62 caches.put(StringIndex.class, new StringIndexCache(this));
63 caches.put(UnValuedDocsCache.class, new UnValuedDocsCache(this));
66 public synchronized void purgeAllCaches() {
70 public synchronized void purge(IndexReader r) {
71 for(Cache c : caches.values()) {
76 public synchronized CacheEntry[] getCacheEntries() {
77 List<CacheEntry> result = new ArrayList<CacheEntry>(17);
78 for(final Map.Entry<Class<?>,Cache> cacheEntry: caches.entrySet()) {
79 final Cache cache = cacheEntry.getValue();
80 final Class<?> cacheType = cacheEntry.getKey();
81 synchronized(cache.readerCache) {
82 for (final Map.Entry<Object,Map<Entry, Object>> readerCacheEntry : cache.readerCache.entrySet()) {
83 final Object readerKey = readerCacheEntry.getKey();
84 if (readerKey == null) continue;
85 final Map<Entry, Object> innerCache = readerCacheEntry.getValue();
86 for (final Map.Entry<Entry, Object> mapEntry : innerCache.entrySet()) {
87 Entry entry = mapEntry.getKey();
88 result.add(new CacheEntryImpl(readerKey, entry.field,
89 cacheType, entry.custom,
90 mapEntry.getValue()));
95 return result.toArray(new CacheEntry[result.size()]);
98 private static final class CacheEntryImpl extends CacheEntry {
99 private final Object readerKey;
100 private final String fieldName;
101 private final Class<?> cacheType;
102 private final Object custom;
103 private final Object value;
104 CacheEntryImpl(Object readerKey, String fieldName,
108 this.readerKey = readerKey;
109 this.fieldName = fieldName;
110 this.cacheType = cacheType;
111 this.custom = custom;
114 // :HACK: for testing.
115 // if (null != locale || SortField.CUSTOM != sortFieldType) {
116 // throw new RuntimeException("Locale/sortFieldType: " + this);
121 public Object getReaderKey() { return readerKey; }
123 public String getFieldName() { return fieldName; }
125 public Class<?> getCacheType() { return cacheType; }
127 public Object getCustom() { return custom; }
129 public Object getValue() { return value; }
133 * Hack: When thrown from a Parser (NUMERIC_UTILS_* ones), this stops
134 * processing terms and returns the current FieldCache
137 static final class StopFillCacheException extends RuntimeException {
140 final static IndexReader.ReaderFinishedListener purgeReader = new IndexReader.ReaderFinishedListener() {
141 // @Override -- not until Java 1.6
142 public void finished(IndexReader reader) {
143 FieldCache.DEFAULT.purge(reader);
147 /** Expert: Internal cache. */
148 abstract static class Cache {
153 Cache(FieldCache wrapper) {
154 this.wrapper = wrapper;
157 final FieldCache wrapper;
159 final Map<Object,Map<Entry,Object>> readerCache = new WeakHashMap<Object,Map<Entry,Object>>();
161 protected abstract Object createValue(IndexReader reader, Entry key)
164 /** Remove this reader from the cache, if present. */
165 public void purge(IndexReader r) {
166 Object readerKey = r.getCoreCacheKey();
167 synchronized(readerCache) {
168 readerCache.remove(readerKey);
172 public Object get(IndexReader reader, Entry key) throws IOException {
173 Map<Entry,Object> innerCache;
175 final Object readerKey = reader.getCoreCacheKey();
176 synchronized (readerCache) {
177 innerCache = readerCache.get(readerKey);
178 if (innerCache == null) {
179 // First time this reader is using FieldCache
180 innerCache = new HashMap<Entry,Object>();
181 readerCache.put(readerKey, innerCache);
182 reader.addReaderFinishedListener(purgeReader);
185 value = innerCache.get(key);
188 value = new CreationPlaceholder();
189 innerCache.put(key, value);
192 if (value instanceof CreationPlaceholder) {
193 synchronized (value) {
194 CreationPlaceholder progress = (CreationPlaceholder) value;
195 if (progress.value == null) {
196 progress.value = createValue(reader, key);
197 synchronized (readerCache) {
198 innerCache.put(key, progress.value);
201 // Only check if key.custom (the parser) is
202 // non-null; else, we check twice for a single
203 // call to FieldCache.getXXX
204 if (key.custom != null && wrapper != null) {
205 final PrintStream infoStream = wrapper.getInfoStream();
206 if (infoStream != null) {
207 printNewInsanity(infoStream, progress.value);
211 return progress.value;
217 private void printNewInsanity(PrintStream infoStream, Object value) {
218 final FieldCacheSanityChecker.Insanity[] insanities = FieldCacheSanityChecker.checkSanity(wrapper);
219 for(int i=0;i<insanities.length;i++) {
220 final FieldCacheSanityChecker.Insanity insanity = insanities[i];
221 final CacheEntry[] entries = insanity.getCacheEntries();
222 for(int j=0;j<entries.length;j++) {
223 if (entries[j].getValue() == value) {
224 // OK this insanity involves our entry
225 infoStream.println("WARNING: new FieldCache insanity created\nDetails: " + insanity.toString());
226 infoStream.println("\nStack:\n");
227 new Throwable().printStackTrace(infoStream);
235 /** Expert: Every composite-key in the internal cache is of this type. */
237 final String field; // which Fieldable
238 final Object custom; // which custom comparator or parser
240 /** Creates one of these objects for a custom comparator/parser. */
241 Entry (String field, Object custom) {
242 this.field = StringHelper.intern(field);
243 this.custom = custom;
246 /** Two of these are equal iff they reference the same field and type. */
248 public boolean equals (Object o) {
249 if (o instanceof Entry) {
250 Entry other = (Entry) o;
251 if (other.field == field) {
252 if (other.custom == null) {
253 if (custom == null) return true;
254 } else if (other.custom.equals (custom)) {
262 /** Composes a hashcode based on the field and type. */
264 public int hashCode() {
265 return field.hashCode() ^ (custom==null ? 0 : custom.hashCode());
270 public byte[] getBytes (IndexReader reader, String field) throws IOException {
271 return getBytes(reader, field, null);
275 public byte[] getBytes(IndexReader reader, String field, ByteParser parser)
277 return (byte[]) caches.get(Byte.TYPE).get(reader, new Entry(field, parser));
280 static final class ByteCache extends Cache {
281 ByteCache(FieldCache wrapper) {
285 protected Object createValue(IndexReader reader, Entry entryKey)
287 Entry entry = entryKey;
288 String field = entry.field;
289 ByteParser parser = (ByteParser) entry.custom;
290 if (parser == null) {
291 return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER);
293 final byte[] retArray = new byte[reader.maxDoc()];
294 TermDocs termDocs = reader.termDocs();
295 TermEnum termEnum = reader.terms (new Term (field));
298 Term term = termEnum.term();
299 if (term==null || term.field() != field) break;
300 byte termval = parser.parseByte(term.text());
301 termDocs.seek (termEnum);
302 while (termDocs.next()) {
303 retArray[termDocs.doc()] = termval;
305 } while (termEnum.next());
306 } catch (StopFillCacheException stop) {
316 public short[] getShorts (IndexReader reader, String field) throws IOException {
317 return getShorts(reader, field, null);
321 public short[] getShorts(IndexReader reader, String field, ShortParser parser)
323 return (short[]) caches.get(Short.TYPE).get(reader, new Entry(field, parser));
326 static final class ShortCache extends Cache {
327 ShortCache(FieldCache wrapper) {
332 protected Object createValue(IndexReader reader, Entry entryKey)
334 Entry entry = entryKey;
335 String field = entry.field;
336 ShortParser parser = (ShortParser) entry.custom;
337 if (parser == null) {
338 return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER);
340 final short[] retArray = new short[reader.maxDoc()];
341 TermDocs termDocs = reader.termDocs();
342 TermEnum termEnum = reader.terms (new Term (field));
345 Term term = termEnum.term();
346 if (term==null || term.field() != field) break;
347 short termval = parser.parseShort(term.text());
348 termDocs.seek (termEnum);
349 while (termDocs.next()) {
350 retArray[termDocs.doc()] = termval;
352 } while (termEnum.next());
353 } catch (StopFillCacheException stop) {
363 public int[] getInts (IndexReader reader, String field) throws IOException {
364 return getInts(reader, field, null);
368 public int[] getInts(IndexReader reader, String field, IntParser parser)
370 return (int[]) caches.get(Integer.TYPE).get(reader, new Entry(field, parser));
373 static final class IntCache extends Cache {
374 IntCache(FieldCache wrapper) {
379 protected Object createValue(IndexReader reader, Entry entryKey)
381 Entry entry = entryKey;
382 String field = entry.field;
383 IntParser parser = (IntParser) entry.custom;
384 if (parser == null) {
386 return wrapper.getInts(reader, field, DEFAULT_INT_PARSER);
387 } catch (NumberFormatException ne) {
388 return wrapper.getInts(reader, field, NUMERIC_UTILS_INT_PARSER);
391 int[] retArray = null;
392 TermDocs termDocs = reader.termDocs();
393 TermEnum termEnum = reader.terms (new Term (field));
396 Term term = termEnum.term();
397 if (term==null || term.field() != field) break;
398 int termval = parser.parseInt(term.text());
399 if (retArray == null) // late init
400 retArray = new int[reader.maxDoc()];
401 termDocs.seek (termEnum);
402 while (termDocs.next()) {
403 retArray[termDocs.doc()] = termval;
405 } while (termEnum.next());
406 } catch (StopFillCacheException stop) {
411 if (retArray == null) // no values
412 retArray = new int[reader.maxDoc()];
417 static final class UnValuedDocsCache extends Cache {
418 UnValuedDocsCache(FieldCache wrapper) {
423 protected Object createValue(IndexReader reader, Entry entryKey)
425 Entry entry = entryKey;
426 String field = entry.field;
428 if (reader.maxDoc() == reader.docFreq(new Term(field))) {
429 return DocIdSet.EMPTY_DOCIDSET;
432 OpenBitSet res = new OpenBitSet(reader.maxDoc());
433 TermDocs termDocs = reader.termDocs();
434 TermEnum termEnum = reader.terms (new Term (field));
437 Term term = termEnum.term();
438 if (term==null || term.field() != field) break;
439 termDocs.seek (termEnum);
440 while (termDocs.next()) {
441 res.fastSet(termDocs.doc());
443 } while (termEnum.next());
448 res.flip(0, reader.maxDoc());
455 public float[] getFloats (IndexReader reader, String field)
457 return getFloats(reader, field, null);
461 public float[] getFloats(IndexReader reader, String field, FloatParser parser)
464 return (float[]) caches.get(Float.TYPE).get(reader, new Entry(field, parser));
467 static final class FloatCache extends Cache {
468 FloatCache(FieldCache wrapper) {
473 protected Object createValue(IndexReader reader, Entry entryKey)
475 Entry entry = entryKey;
476 String field = entry.field;
477 FloatParser parser = (FloatParser) entry.custom;
478 if (parser == null) {
480 return wrapper.getFloats(reader, field, DEFAULT_FLOAT_PARSER);
481 } catch (NumberFormatException ne) {
482 return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER);
485 float[] retArray = null;
486 TermDocs termDocs = reader.termDocs();
487 TermEnum termEnum = reader.terms (new Term (field));
490 Term term = termEnum.term();
491 if (term==null || term.field() != field) break;
492 float termval = parser.parseFloat(term.text());
493 if (retArray == null) // late init
494 retArray = new float[reader.maxDoc()];
495 termDocs.seek (termEnum);
496 while (termDocs.next()) {
497 retArray[termDocs.doc()] = termval;
499 } while (termEnum.next());
500 } catch (StopFillCacheException stop) {
505 if (retArray == null) // no values
506 retArray = new float[reader.maxDoc()];
512 public long[] getLongs(IndexReader reader, String field) throws IOException {
513 return getLongs(reader, field, null);
517 public long[] getLongs(IndexReader reader, String field, FieldCache.LongParser parser)
519 return (long[]) caches.get(Long.TYPE).get(reader, new Entry(field, parser));
522 static final class LongCache extends Cache {
523 LongCache(FieldCache wrapper) {
528 protected Object createValue(IndexReader reader, Entry entry)
530 String field = entry.field;
531 FieldCache.LongParser parser = (FieldCache.LongParser) entry.custom;
532 if (parser == null) {
534 return wrapper.getLongs(reader, field, DEFAULT_LONG_PARSER);
535 } catch (NumberFormatException ne) {
536 return wrapper.getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER);
539 long[] retArray = null;
540 TermDocs termDocs = reader.termDocs();
541 TermEnum termEnum = reader.terms (new Term(field));
544 Term term = termEnum.term();
545 if (term==null || term.field() != field) break;
546 long termval = parser.parseLong(term.text());
547 if (retArray == null) // late init
548 retArray = new long[reader.maxDoc()];
549 termDocs.seek (termEnum);
550 while (termDocs.next()) {
551 retArray[termDocs.doc()] = termval;
553 } while (termEnum.next());
554 } catch (StopFillCacheException stop) {
559 if (retArray == null) // no values
560 retArray = new long[reader.maxDoc()];
566 public double[] getDoubles(IndexReader reader, String field)
568 return getDoubles(reader, field, null);
572 public double[] getDoubles(IndexReader reader, String field, FieldCache.DoubleParser parser)
574 return (double[]) caches.get(Double.TYPE).get(reader, new Entry(field, parser));
577 static final class DoubleCache extends Cache {
578 DoubleCache(FieldCache wrapper) {
583 protected Object createValue(IndexReader reader, Entry entryKey)
585 Entry entry = entryKey;
586 String field = entry.field;
587 FieldCache.DoubleParser parser = (FieldCache.DoubleParser) entry.custom;
588 if (parser == null) {
590 return wrapper.getDoubles(reader, field, DEFAULT_DOUBLE_PARSER);
591 } catch (NumberFormatException ne) {
592 return wrapper.getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER);
595 double[] retArray = null;
596 TermDocs termDocs = reader.termDocs();
597 TermEnum termEnum = reader.terms (new Term (field));
600 Term term = termEnum.term();
601 if (term==null || term.field() != field) break;
602 double termval = parser.parseDouble(term.text());
603 if (retArray == null) // late init
604 retArray = new double[reader.maxDoc()];
605 termDocs.seek (termEnum);
606 while (termDocs.next()) {
607 retArray[termDocs.doc()] = termval;
609 } while (termEnum.next());
610 } catch (StopFillCacheException stop) {
615 if (retArray == null) // no values
616 retArray = new double[reader.maxDoc()];
622 public String[] getStrings(IndexReader reader, String field)
624 return (String[]) caches.get(String.class).get(reader, new Entry(field, (Parser)null));
627 static final class StringCache extends Cache {
628 StringCache(FieldCache wrapper) {
633 protected Object createValue(IndexReader reader, Entry entryKey)
635 String field = StringHelper.intern(entryKey.field);
636 final String[] retArray = new String[reader.maxDoc()];
637 TermDocs termDocs = reader.termDocs();
638 TermEnum termEnum = reader.terms (new Term (field));
639 final int termCountHardLimit = reader.maxDoc();
643 if (termCount++ == termCountHardLimit) {
644 // app is misusing the API (there is more than
645 // one term per doc); in this case we make best
646 // effort to load what we can (see LUCENE-2142)
650 Term term = termEnum.term();
651 if (term==null || term.field() != field) break;
652 String termval = term.text();
653 termDocs.seek (termEnum);
654 while (termDocs.next()) {
655 retArray[termDocs.doc()] = termval;
657 } while (termEnum.next());
667 public StringIndex getStringIndex(IndexReader reader, String field)
669 return (StringIndex) caches.get(StringIndex.class).get(reader, new Entry(field, (Parser)null));
672 static final class StringIndexCache extends Cache {
673 StringIndexCache(FieldCache wrapper) {
678 protected Object createValue(IndexReader reader, Entry entryKey)
680 String field = StringHelper.intern(entryKey.field);
681 final int[] retArray = new int[reader.maxDoc()];
682 String[] mterms = new String[reader.maxDoc()+1];
683 TermDocs termDocs = reader.termDocs();
684 TermEnum termEnum = reader.terms (new Term (field));
685 int t = 0; // current term number
687 // an entry for documents that have no terms in this field
688 // should a document with no terms be at top or bottom?
689 // this puts them at the top - if it is changed, FieldDocSortedHitQueue
690 // needs to change as well.
695 Term term = termEnum.term();
696 if (term==null || term.field() != field || t >= mterms.length) break;
699 mterms[t] = term.text();
701 termDocs.seek (termEnum);
702 while (termDocs.next()) {
703 retArray[termDocs.doc()] = t;
707 } while (termEnum.next());
714 // if there are no terms, make the term array
715 // have a single null entry
716 mterms = new String[1];
717 } else if (t < mterms.length) {
718 // if there are less terms than documents,
719 // trim off the dead array space
720 String[] terms = new String[t];
721 System.arraycopy (mterms, 0, terms, 0, t);
725 StringIndex value = new StringIndex (retArray, mterms);
730 private volatile PrintStream infoStream;
732 public void setInfoStream(PrintStream stream) {
736 public PrintStream getInfoStream() {
740 public DocIdSet getUnValuedDocs(IndexReader reader, String field)
742 return (DocIdSet) caches.get(UnValuedDocsCache.class).get(reader, new Entry(field, null));