--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.SortedSet;
+
+import org.apache.lucene.analysis.*;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestTermVectorsReader extends LuceneTestCase {
+ //Must be lexicographically sorted, will do in setup, versus trying to maintain here
+ private String[] testFields = {"f1", "f2", "f3", "f4"};
+ private boolean[] testFieldsStorePos = {true, false, true, false};
+ private boolean[] testFieldsStoreOff = {true, false, false, true};
+ private String[] testTerms = {"this", "is", "a", "test"};
+ private int[][] positions = new int[testTerms.length][];
+ private TermVectorOffsetInfo[][] offsets = new TermVectorOffsetInfo[testTerms.length][];
+ private Directory dir;
+ private String seg;
+ private FieldInfos fieldInfos = new FieldInfos();
+ private static int TERM_FREQ = 3;
+
+ private class TestToken implements Comparable<TestToken> {
+ String text;
+ int pos;
+ int startOffset;
+ int endOffset;
+ public int compareTo(TestToken other) {
+ return pos - other.pos;
+ }
+ }
+
+ TestToken[] tokens = new TestToken[testTerms.length * TERM_FREQ];
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ /*
+ for (int i = 0; i < testFields.length; i++) {
+ fieldInfos.add(testFields[i], true, true, testFieldsStorePos[i], testFieldsStoreOff[i]);
+ }
+ */
+
+ Arrays.sort(testTerms);
+ int tokenUpto = 0;
+ for (int i = 0; i < testTerms.length; i++) {
+ positions[i] = new int[TERM_FREQ];
+ offsets[i] = new TermVectorOffsetInfo[TERM_FREQ];
+ // first position must be 0
+ for (int j = 0; j < TERM_FREQ; j++) {
+ // positions are always sorted in increasing order
+ positions[i][j] = (int) (j * 10 + Math.random() * 10);
+ // offsets are always sorted in increasing order
+ offsets[i][j] = new TermVectorOffsetInfo(j * 10, j * 10 + testTerms[i].length());
+ TestToken token = tokens[tokenUpto++] = new TestToken();
+ token.text = testTerms[i];
+ token.pos = positions[i][j];
+ token.startOffset = offsets[i][j].getStartOffset();
+ token.endOffset = offsets[i][j].getEndOffset();
+ }
+ }
+ Arrays.sort(tokens);
+
+ dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MyAnalyzer()).setMaxBufferedDocs(-1).setMergePolicy(newLogMergePolicy(false, 10)));
+
+ Document doc = new Document();
+ for(int i=0;i<testFields.length;i++) {
+ final Field.TermVector tv;
+ if (testFieldsStorePos[i] && testFieldsStoreOff[i])
+ tv = Field.TermVector.WITH_POSITIONS_OFFSETS;
+ else if (testFieldsStorePos[i] && !testFieldsStoreOff[i])
+ tv = Field.TermVector.WITH_POSITIONS;
+ else if (!testFieldsStorePos[i] && testFieldsStoreOff[i])
+ tv = Field.TermVector.WITH_OFFSETS;
+ else
+ tv = Field.TermVector.YES;
+ doc.add(new Field(testFields[i], "", Field.Store.NO, Field.Index.ANALYZED, tv));
+ }
+
+ //Create 5 documents for testing, they all have the same
+ //terms
+ for(int j=0;j<5;j++)
+ writer.addDocument(doc);
+ writer.commit();
+ seg = writer.newestSegment().name;
+ writer.close();
+
+ fieldInfos = new FieldInfos(dir, IndexFileNames.segmentFileName(seg, IndexFileNames.FIELD_INFOS_EXTENSION));
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ dir.close();
+ super.tearDown();
+ }
+
+ private class MyTokenStream extends TokenStream {
+ private int tokenUpto;
+
+ private final CharTermAttribute termAtt;
+ private final PositionIncrementAttribute posIncrAtt;
+ private final OffsetAttribute offsetAtt;
+
+ public MyTokenStream() {
+ termAtt = addAttribute(CharTermAttribute.class);
+ posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ offsetAtt = addAttribute(OffsetAttribute.class);
+ }
+
+ @Override
+ public boolean incrementToken() {
+ if (tokenUpto >= tokens.length)
+ return false;
+ else {
+ final TestToken testToken = tokens[tokenUpto++];
+ clearAttributes();
+ termAtt.append(testToken.text);
+ offsetAtt.setOffset(testToken.startOffset, testToken.endOffset);
+ if (tokenUpto > 1) {
+ posIncrAtt.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos);
+ } else {
+ posIncrAtt.setPositionIncrement(testToken.pos+1);
+ }
+ return true;
+ }
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ this.tokenUpto = 0;
+ }
+ }
+
+ private class MyAnalyzer extends Analyzer {
+ @Override
+ public TokenStream tokenStream(String fieldName, Reader reader) {
+ return new MyTokenStream();
+ }
+ }
+
+ public void test() throws IOException {
+ //Check to see the files were created properly in setup
+ assertTrue(dir.fileExists(IndexFileNames.segmentFileName(seg, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)));
+ assertTrue(dir.fileExists(IndexFileNames.segmentFileName(seg, IndexFileNames.VECTORS_INDEX_EXTENSION)));
+ }
+
+ public void testReader() throws IOException {
+ TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);
+ for (int j = 0; j < 5; j++) {
+ TermFreqVector vector = reader.get(j, testFields[0]);
+ assertTrue(vector != null);
+ String[] terms = vector.getTerms();
+ assertTrue(terms != null);
+ assertTrue(terms.length == testTerms.length);
+ for (int i = 0; i < terms.length; i++) {
+ String term = terms[i];
+ //System.out.println("Term: " + term);
+ assertTrue(term.equals(testTerms[i]));
+ }
+ }
+ reader.close();
+ }
+
+ public void testPositionReader() throws IOException {
+ TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);
+ TermPositionVector vector;
+ String[] terms;
+ vector = (TermPositionVector) reader.get(0, testFields[0]);
+ assertTrue(vector != null);
+ terms = vector.getTerms();
+ assertTrue(terms != null);
+ assertTrue(terms.length == testTerms.length);
+ for (int i = 0; i < terms.length; i++) {
+ String term = terms[i];
+ //System.out.println("Term: " + term);
+ assertTrue(term.equals(testTerms[i]));
+ int[] positions = vector.getTermPositions(i);
+ assertTrue(positions != null);
+ assertTrue(positions.length == this.positions[i].length);
+ for (int j = 0; j < positions.length; j++) {
+ int position = positions[j];
+ assertTrue(position == this.positions[i][j]);
+ }
+ TermVectorOffsetInfo[] offset = vector.getOffsets(i);
+ assertTrue(offset != null);
+ assertTrue(offset.length == this.offsets[i].length);
+ for (int j = 0; j < offset.length; j++) {
+ TermVectorOffsetInfo termVectorOffsetInfo = offset[j];
+ assertTrue(termVectorOffsetInfo.equals(offsets[i][j]));
+ }
+ }
+
+ TermFreqVector freqVector = reader.get(0, testFields[1]); //no pos, no offset
+ assertTrue(freqVector != null);
+ assertTrue(freqVector instanceof TermPositionVector == false);
+ terms = freqVector.getTerms();
+ assertTrue(terms != null);
+ assertTrue(terms.length == testTerms.length);
+ for (int i = 0; i < terms.length; i++) {
+ String term = terms[i];
+ //System.out.println("Term: " + term);
+ assertTrue(term.equals(testTerms[i]));
+ }
+ reader.close();
+ }
+
+ public void testOffsetReader() throws IOException {
+ TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);
+ TermPositionVector vector = (TermPositionVector) reader.get(0, testFields[0]);
+ assertTrue(vector != null);
+ String[] terms = vector.getTerms();
+ assertTrue(terms != null);
+ assertTrue(terms.length == testTerms.length);
+ for (int i = 0; i < terms.length; i++) {
+ String term = terms[i];
+ //System.out.println("Term: " + term);
+ assertTrue(term.equals(testTerms[i]));
+ int[] positions = vector.getTermPositions(i);
+ assertTrue(positions != null);
+ assertTrue(positions.length == this.positions[i].length);
+ for (int j = 0; j < positions.length; j++) {
+ int position = positions[j];
+ assertTrue(position == this.positions[i][j]);
+ }
+ TermVectorOffsetInfo[] offset = vector.getOffsets(i);
+ assertTrue(offset != null);
+ assertTrue(offset.length == this.offsets[i].length);
+ for (int j = 0; j < offset.length; j++) {
+ TermVectorOffsetInfo termVectorOffsetInfo = offset[j];
+ assertTrue(termVectorOffsetInfo.equals(offsets[i][j]));
+ }
+ }
+ reader.close();
+ }
+
+ public void testMapper() throws IOException {
+ TermVectorsReader reader = new TermVectorsReader(dir, seg, fieldInfos);
+ SortedTermVectorMapper mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
+ reader.get(0, mapper);
+ SortedSet<TermVectorEntry> set = mapper.getTermVectorEntrySet();
+ assertTrue("set is null and it shouldn't be", set != null);
+ //three fields, 4 terms, all terms are the same
+ assertTrue("set Size: " + set.size() + " is not: " + 4, set.size() == 4);
+ //Check offsets and positions
+ for (Iterator<TermVectorEntry> iterator = set.iterator(); iterator.hasNext();) {
+ TermVectorEntry tve = iterator.next();
+ assertTrue("tve is null and it shouldn't be", tve != null);
+ assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() != null);
+ assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() != null);
+
+ }
+
+ mapper = new SortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
+ reader.get(1, mapper);
+ set = mapper.getTermVectorEntrySet();
+ assertTrue("set is null and it shouldn't be", set != null);
+ //three fields, 4 terms, all terms are the same
+ assertTrue("set Size: " + set.size() + " is not: " + 4, set.size() == 4);
+ //Should have offsets and positions b/c we are munging all the fields together
+ for (Iterator<TermVectorEntry> iterator = set.iterator(); iterator.hasNext();) {
+ TermVectorEntry tve = iterator.next();
+ assertTrue("tve is null and it shouldn't be", tve != null);
+ assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() != null);
+ assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() != null);
+
+ }
+
+
+ FieldSortedTermVectorMapper fsMapper = new FieldSortedTermVectorMapper(new TermVectorEntryFreqSortedComparator());
+ reader.get(0, fsMapper);
+ Map<String,SortedSet<TermVectorEntry>> map = fsMapper.getFieldToTerms();
+ assertTrue("map Size: " + map.size() + " is not: " + testFields.length, map.size() == testFields.length);
+ for (Map.Entry<String,SortedSet<TermVectorEntry>> entry : map.entrySet()) {
+ SortedSet<TermVectorEntry> sortedSet = entry.getValue();
+ assertTrue("sortedSet Size: " + sortedSet.size() + " is not: " + 4, sortedSet.size() == 4);
+ for (final TermVectorEntry tve : sortedSet) {
+ assertTrue("tve is null and it shouldn't be", tve != null);
+ //Check offsets and positions.
+ assertTrue("tve is null and it shouldn't be", tve != null);
+ String field = tve.getField();
+ if (field.equals(testFields[0])) {
+ //should have offsets
+
+ assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() != null);
+ assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() != null);
+ }
+ else if (field.equals(testFields[1])) {
+ //should not have offsets
+
+ assertTrue("tve.getOffsets() is not null and it shouldn't be", tve.getOffsets() == null);
+ assertTrue("tve.getPositions() is not null and it shouldn't be", tve.getPositions() == null);
+ }
+ }
+ }
+ //Try mapper that ignores offs and positions
+ fsMapper = new FieldSortedTermVectorMapper(true, true, new TermVectorEntryFreqSortedComparator());
+ reader.get(0, fsMapper);
+ map = fsMapper.getFieldToTerms();
+ assertTrue("map Size: " + map.size() + " is not: " + testFields.length, map.size() == testFields.length);
+ for (final Map.Entry<String,SortedSet<TermVectorEntry>> entry : map.entrySet()) {
+ SortedSet<TermVectorEntry> sortedSet = entry.getValue();
+ assertTrue("sortedSet Size: " + sortedSet.size() + " is not: " + 4, sortedSet.size() == 4);
+ for (final TermVectorEntry tve : sortedSet) {
+ assertTrue("tve is null and it shouldn't be", tve != null);
+ //Check offsets and positions.
+ assertTrue("tve is null and it shouldn't be", tve != null);
+ String field = tve.getField();
+ if (field.equals(testFields[0])) {
+ //should have offsets
+
+ assertTrue("tve.getOffsets() is null and it shouldn't be", tve.getOffsets() == null);
+ assertTrue("tve.getPositions() is null and it shouldn't be", tve.getPositions() == null);
+ }
+ else if (field.equals(testFields[1])) {
+ //should not have offsets
+
+ assertTrue("tve.getOffsets() is not null and it shouldn't be", tve.getOffsets() == null);
+ assertTrue("tve.getPositions() is not null and it shouldn't be", tve.getPositions() == null);
+ }
+ }
+ }
+
+ // test setDocumentNumber()
+ IndexReader ir = IndexReader.open(dir, true);
+ DocNumAwareMapper docNumAwareMapper = new DocNumAwareMapper();
+ assertEquals(-1, docNumAwareMapper.getDocumentNumber());
+
+ ir.getTermFreqVector(0, docNumAwareMapper);
+ assertEquals(0, docNumAwareMapper.getDocumentNumber());
+ docNumAwareMapper.setDocumentNumber(-1);
+
+ ir.getTermFreqVector(1, docNumAwareMapper);
+ assertEquals(1, docNumAwareMapper.getDocumentNumber());
+ docNumAwareMapper.setDocumentNumber(-1);
+
+ ir.getTermFreqVector(0, "f1", docNumAwareMapper);
+ assertEquals(0, docNumAwareMapper.getDocumentNumber());
+ docNumAwareMapper.setDocumentNumber(-1);
+
+ ir.getTermFreqVector(1, "f2", docNumAwareMapper);
+ assertEquals(1, docNumAwareMapper.getDocumentNumber());
+ docNumAwareMapper.setDocumentNumber(-1);
+
+ ir.getTermFreqVector(0, "f1", docNumAwareMapper);
+ assertEquals(0, docNumAwareMapper.getDocumentNumber());
+
+ ir.close();
+ reader.close();
+ }
+
+
+ /**
+ * Make sure exceptions and bad params are handled appropriately
+ */
+ public void testBadParams() throws IOException {
+ TermVectorsReader reader = null;
+ try {
+ reader = new TermVectorsReader(dir, seg, fieldInfos);
+ //Bad document number, good field number
+ reader.get(50, testFields[0]);
+ fail();
+ } catch (IOException e) {
+ // expected exception
+ } finally {
+ reader.close();
+ }
+ try {
+ reader = new TermVectorsReader(dir, seg, fieldInfos);
+ //Bad document number, no field
+ reader.get(50);
+ fail();
+ } catch (IOException e) {
+ // expected exception
+ } finally {
+ reader.close();
+ }
+ try {
+ reader = new TermVectorsReader(dir, seg, fieldInfos);
+ //good document number, bad field number
+ TermFreqVector vector = reader.get(0, "f50");
+ assertTrue(vector == null);
+ reader.close();
+ } catch (IOException e) {
+ fail();
+ } finally {
+ reader.close();
+ }
+ }
+
+
+ public static class DocNumAwareMapper extends TermVectorMapper {
+
+ public DocNumAwareMapper() {
+ }
+
+ private int documentNumber = -1;
+
+ @Override
+ public void setExpectations(String field, int numTerms, boolean storeOffsets, boolean storePositions) {
+ if (documentNumber == -1) {
+ throw new RuntimeException("Documentnumber should be set at this point!");
+ }
+ }
+
+ @Override
+ public void map(String term, int frequency, TermVectorOffsetInfo[] offsets, int[] positions) {
+ if (documentNumber == -1) {
+ throw new RuntimeException("Documentnumber should be set at this point!");
+ }
+ }
+
+ public int getDocumentNumber() {
+ return documentNumber;
+ }
+
+ @Override
+ public void setDocumentNumber(int documentNumber) {
+ this.documentNumber = documentNumber;
+ }
+ }
+}
fnp / pylucene.git / blobdiff