1 package org.apache.lucene.index;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import java.io.IOException;
21 import java.io.Reader;
23 import org.apache.lucene.analysis.Analyzer;
24 import org.apache.lucene.analysis.LowerCaseTokenizer;
25 import org.apache.lucene.analysis.TokenFilter;
26 import org.apache.lucene.analysis.TokenStream;
27 import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
28 import org.apache.lucene.document.Document;
29 import org.apache.lucene.document.Field.Index;
30 import org.apache.lucene.document.Field.Store;
31 import org.apache.lucene.store.IndexInput;
32 import org.apache.lucene.store.RAMDirectory;
33 import org.apache.lucene.util.LuceneTestCase;
36 * This testcase tests whether multi-level skipping is being used
37 * to reduce I/O while skipping through posting lists.
39 * Skipping in general is already covered by several other
43 public class TestMultiLevelSkipList extends LuceneTestCase {
45 public void setUp() throws Exception {
47 PayloadFilter.count = 0;
50 public void testSimpleSkip() throws IOException {
51 RAMDirectory dir = new RAMDirectory();
52 IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new PayloadAnalyzer()).setMergePolicy(newLogMergePolicy()));
53 Term term = new Term("test", "a");
54 for (int i = 0; i < 5000; i++) {
55 Document d1 = new Document();
56 d1.add(newField(term.field(), term.text(), Store.NO, Index.ANALYZED));
57 writer.addDocument(d1);
63 IndexReader reader = SegmentReader.getOnlySegmentReader(dir);
64 SegmentTermPositions tp = (SegmentTermPositions) reader.termPositions();
65 tp.freqStream = new CountingStream(tp.freqStream);
67 for (int i = 0; i < 2; i++) {
71 checkSkipTo(tp, 14, 185); // no skips
72 checkSkipTo(tp, 17, 190); // one skip on level 0
73 checkSkipTo(tp, 287, 200); // one skip on level 1, two on level 0
75 // this test would fail if we had only one skip level,
76 // because than more bytes would be read from the freqStream
77 checkSkipTo(tp, 4800, 250);// one skip on level 2
81 public void checkSkipTo(TermPositions tp, int target, int maxCounter) throws IOException {
83 if (maxCounter < counter) {
84 fail("Too many bytes read: " + counter + " vs " + maxCounter);
87 assertEquals("Wrong document " + tp.doc() + " after skipTo target " + target, target, tp.doc());
88 assertEquals("Frequency is not 1: " + tp.freq(), 1,tp.freq());
90 byte[] b = new byte[1];
92 assertEquals("Wrong payload for the target " + target + ": " + b[0], (byte) target, b[0]);
95 private static class PayloadAnalyzer extends Analyzer {
97 public TokenStream tokenStream(String fieldName, Reader reader) {
98 return new PayloadFilter(new LowerCaseTokenizer(TEST_VERSION_CURRENT, reader));
103 private static class PayloadFilter extends TokenFilter {
104 static int count = 0;
106 PayloadAttribute payloadAtt;
108 protected PayloadFilter(TokenStream input) {
110 payloadAtt = addAttribute(PayloadAttribute.class);
114 public boolean incrementToken() throws IOException {
115 boolean hasNext = input.incrementToken();
117 payloadAtt.setPayload(new Payload(new byte[] { (byte) count++ }));
124 private int counter = 0;
126 // Simply extends IndexInput in a way that we are able to count the number
128 class CountingStream extends IndexInput {
129 private IndexInput input;
131 CountingStream(IndexInput input) {
136 public byte readByte() throws IOException {
137 TestMultiLevelSkipList.this.counter++;
138 return this.input.readByte();
142 public void readBytes(byte[] b, int offset, int len) throws IOException {
143 TestMultiLevelSkipList.this.counter += len;
144 this.input.readBytes(b, offset, len);
148 public void close() throws IOException {
153 public long getFilePointer() {
154 return this.input.getFilePointer();
158 public void seek(long pos) throws IOException {
159 this.input.seek(pos);
163 public long length() {
164 return this.input.length();
168 public Object clone() {
169 return new CountingStream((IndexInput) this.input.clone());