+++ /dev/null
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.io.StringReader;
-
-import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CachingTokenFilter;
-import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.TeeSinkTokenFilter;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceAnalyzer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.MockDirectoryWrapper;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.LuceneTestCase;
-
-/** tests for writing term vectors */
-public class TestTermVectorsWriter extends LuceneTestCase {
- // LUCENE-1442
- public void testDoubleOffsetCounting() throws Exception {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document doc = new Document();
- Field f = newField("field", "abcd", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f);
- doc.add(f);
- Field f2 = newField("field", "", Field.Store.NO, Field.Index.NOT_ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f2);
- doc.add(f);
- w.addDocument(doc);
- w.close();
-
- IndexReader r = IndexReader.open(dir, true);
- TermVectorOffsetInfo[] termOffsets = ((TermPositionVector) r.getTermFreqVector(0, "field")).getOffsets(0);
-
- // Token "" occurred once
- assertEquals(1, termOffsets.length);
- assertEquals(8, termOffsets[0].getStartOffset());
- assertEquals(8, termOffsets[0].getEndOffset());
-
- // Token "abcd" occurred three times
- termOffsets = ((TermPositionVector) r.getTermFreqVector(0, "field")).getOffsets(1);
- assertEquals(3, termOffsets.length);
- assertEquals(0, termOffsets[0].getStartOffset());
- assertEquals(4, termOffsets[0].getEndOffset());
- assertEquals(4, termOffsets[1].getStartOffset());
- assertEquals(8, termOffsets[1].getEndOffset());
- assertEquals(8, termOffsets[2].getStartOffset());
- assertEquals(12, termOffsets[2].getEndOffset());
- r.close();
- dir.close();
- }
-
- // LUCENE-1442
- public void testDoubleOffsetCounting2() throws Exception {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document doc = new Document();
- Field f = newField("field", "abcd", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f);
- doc.add(f);
- w.addDocument(doc);
- w.close();
-
- IndexReader r = IndexReader.open(dir, true);
- TermVectorOffsetInfo[] termOffsets = ((TermPositionVector) r.getTermFreqVector(0, "field")).getOffsets(0);
- assertEquals(2, termOffsets.length);
- assertEquals(0, termOffsets[0].getStartOffset());
- assertEquals(4, termOffsets[0].getEndOffset());
- assertEquals(5, termOffsets[1].getStartOffset());
- assertEquals(9, termOffsets[1].getEndOffset());
- r.close();
- dir.close();
- }
-
- // LUCENE-1448
- public void testEndOffsetPositionCharAnalyzer() throws Exception {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document doc = new Document();
- Field f = newField("field", "abcd ", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f);
- doc.add(f);
- w.addDocument(doc);
- w.close();
-
- IndexReader r = IndexReader.open(dir, true);
- TermVectorOffsetInfo[] termOffsets = ((TermPositionVector) r.getTermFreqVector(0, "field")).getOffsets(0);
- assertEquals(2, termOffsets.length);
- assertEquals(0, termOffsets[0].getStartOffset());
- assertEquals(4, termOffsets[0].getEndOffset());
- assertEquals(8, termOffsets[1].getStartOffset());
- assertEquals(12, termOffsets[1].getEndOffset());
- r.close();
- dir.close();
- }
-
- // LUCENE-1448
- public void testEndOffsetPositionWithCachingTokenFilter() throws Exception {
- Directory dir = newDirectory();
- Analyzer analyzer = new MockAnalyzer(random);
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
- Document doc = new Document();
- TokenStream stream = analyzer.tokenStream("field", new StringReader("abcd "));
- stream.reset(); // TODO: wierd to reset before wrapping with CachingTokenFilter... correct?
- stream = new CachingTokenFilter(stream);
- Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f);
- doc.add(f);
- w.addDocument(doc);
- w.close();
-
- IndexReader r = IndexReader.open(dir, true);
- TermVectorOffsetInfo[] termOffsets = ((TermPositionVector) r.getTermFreqVector(0, "field")).getOffsets(0);
- assertEquals(2, termOffsets.length);
- assertEquals(0, termOffsets[0].getStartOffset());
- assertEquals(4, termOffsets[0].getEndOffset());
- assertEquals(8, termOffsets[1].getStartOffset());
- assertEquals(12, termOffsets[1].getEndOffset());
- r.close();
- dir.close();
- }
-
- // LUCENE-1448
- public void testEndOffsetPositionWithTeeSinkTokenFilter() throws Exception {
- MockDirectoryWrapper dir = newDirectory();
- Analyzer analyzer = new WhitespaceAnalyzer(TEST_VERSION_CURRENT);
- IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer));
- Document doc = new Document();
- TeeSinkTokenFilter tee = new TeeSinkTokenFilter(analyzer.tokenStream("field", new StringReader("abcd ")));
- TokenStream sink = tee.newSinkTokenStream();
- Field f1 = new Field("field", tee, Field.TermVector.WITH_POSITIONS_OFFSETS);
- Field f2 = new Field("field", sink, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f1);
- doc.add(f2);
- w.addDocument(doc);
- w.close();
-
- IndexReader r = IndexReader.open(dir, true);
- TermVectorOffsetInfo[] termOffsets = ((TermPositionVector) r.getTermFreqVector(0, "field")).getOffsets(0);
- assertEquals(2, termOffsets.length);
- assertEquals(0, termOffsets[0].getStartOffset());
- assertEquals(4, termOffsets[0].getEndOffset());
- assertEquals(8, termOffsets[1].getStartOffset());
- assertEquals(12, termOffsets[1].getEndOffset());
- r.close();
- dir.close();
- }
-
- // LUCENE-1448
- public void testEndOffsetPositionStopFilter() throws Exception {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new StandardAnalyzer(TEST_VERSION_CURRENT)));
- Document doc = new Document();
- Field f = newField("field", "abcd the", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f);
- doc.add(f);
- w.addDocument(doc);
- w.close();
-
- IndexReader r = IndexReader.open(dir, true);
- TermVectorOffsetInfo[] termOffsets = ((TermPositionVector) r.getTermFreqVector(0, "field")).getOffsets(0);
- assertEquals(2, termOffsets.length);
- assertEquals(0, termOffsets[0].getStartOffset());
- assertEquals(4, termOffsets[0].getEndOffset());
- assertEquals(9, termOffsets[1].getStartOffset());
- assertEquals(13, termOffsets[1].getEndOffset());
- r.close();
- dir.close();
- }
-
- // LUCENE-1448
- public void testEndOffsetPositionStandard() throws Exception {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document doc = new Document();
- Field f = newField("field", "abcd the ", Field.Store.NO,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- Field f2 = newField("field", "crunch man", Field.Store.NO,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f);
- doc.add(f2);
- w.addDocument(doc);
- w.close();
-
- IndexReader r = IndexReader.open(dir, true);
- TermPositionVector tpv = ((TermPositionVector) r.getTermFreqVector(0, "field"));
- TermVectorOffsetInfo[] termOffsets = tpv.getOffsets(0);
- assertEquals(1, termOffsets.length);
- assertEquals(0, termOffsets[0].getStartOffset());
- assertEquals(4, termOffsets[0].getEndOffset());
- termOffsets = tpv.getOffsets(1);
- assertEquals(11, termOffsets[0].getStartOffset());
- assertEquals(17, termOffsets[0].getEndOffset());
- termOffsets = tpv.getOffsets(2);
- assertEquals(18, termOffsets[0].getStartOffset());
- assertEquals(21, termOffsets[0].getEndOffset());
- r.close();
- dir.close();
- }
-
- // LUCENE-1448
- public void testEndOffsetPositionStandardEmptyField() throws Exception {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document doc = new Document();
- Field f = newField("field", "", Field.Store.NO,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- Field f2 = newField("field", "crunch man", Field.Store.NO,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f);
- doc.add(f2);
- w.addDocument(doc);
- w.close();
-
- IndexReader r = IndexReader.open(dir, true);
- TermPositionVector tpv = ((TermPositionVector) r.getTermFreqVector(0, "field"));
- TermVectorOffsetInfo[] termOffsets = tpv.getOffsets(0);
- assertEquals(1, termOffsets.length);
- assertEquals(1, termOffsets[0].getStartOffset());
- assertEquals(7, termOffsets[0].getEndOffset());
- termOffsets = tpv.getOffsets(1);
- assertEquals(8, termOffsets[0].getStartOffset());
- assertEquals(11, termOffsets[0].getEndOffset());
- r.close();
- dir.close();
- }
-
- // LUCENE-1448
- public void testEndOffsetPositionStandardEmptyField2() throws Exception {
- Directory dir = newDirectory();
- IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document doc = new Document();
-
- Field f = newField("field", "abcd", Field.Store.NO,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f);
- doc.add(newField("field", "", Field.Store.NO, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
-
- Field f2 = newField("field", "crunch", Field.Store.NO,
- Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS);
- doc.add(f2);
-
- w.addDocument(doc);
- w.close();
-
- IndexReader r = IndexReader.open(dir, true);
- TermPositionVector tpv = ((TermPositionVector) r.getTermFreqVector(0, "field"));
- TermVectorOffsetInfo[] termOffsets = tpv.getOffsets(0);
- assertEquals(1, termOffsets.length);
- assertEquals(0, termOffsets[0].getStartOffset());
- assertEquals(4, termOffsets[0].getEndOffset());
- termOffsets = tpv.getOffsets(1);
- assertEquals(6, termOffsets[0].getStartOffset());
- assertEquals(12, termOffsets[0].getEndOffset());
- r.close();
- dir.close();
- }
-
- // LUCENE-1168
- public void testTermVectorCorruption() throws IOException {
-
- Directory dir = newDirectory();
- for(int iter=0;iter<2;iter++) {
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMaxBufferedDocs(2).setRAMBufferSizeMB(
- IndexWriterConfig.DISABLE_AUTO_FLUSH).setMergeScheduler(
- new SerialMergeScheduler()).setMergePolicy(
- new LogDocMergePolicy()));
-
- Document document = new Document();
-
- Field storedField = newField("stored", "stored", Field.Store.YES,
- Field.Index.NO);
- document.add(storedField);
- writer.addDocument(document);
- writer.addDocument(document);
-
- document = new Document();
- document.add(storedField);
- Field termVectorField = newField("termVector", "termVector",
- Field.Store.NO, Field.Index.NOT_ANALYZED,
- Field.TermVector.WITH_POSITIONS_OFFSETS);
-
- document.add(termVectorField);
- writer.addDocument(document);
- writer.optimize();
- writer.close();
-
- IndexReader reader = IndexReader.open(dir, true);
- for(int i=0;i<reader.numDocs();i++) {
- reader.document(i);
- reader.getTermFreqVectors(i);
- }
- reader.close();
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
- new MockAnalyzer(random)).setMaxBufferedDocs(2)
- .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
- .setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(
- new LogDocMergePolicy()));
-
- Directory[] indexDirs = {new MockDirectoryWrapper(random, new RAMDirectory(dir))};
- writer.addIndexes(indexDirs);
- writer.optimize();
- writer.close();
- }
- dir.close();
- }
-
- // LUCENE-1168
- public void testTermVectorCorruption2() throws IOException {
- Directory dir = newDirectory();
- for(int iter=0;iter<2;iter++) {
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMaxBufferedDocs(2).setRAMBufferSizeMB(
- IndexWriterConfig.DISABLE_AUTO_FLUSH).setMergeScheduler(
- new SerialMergeScheduler()).setMergePolicy(
- new LogDocMergePolicy()));
-
- Document document = new Document();
-
- Field storedField = newField("stored", "stored", Field.Store.YES,
- Field.Index.NO);
- document.add(storedField);
- writer.addDocument(document);
- writer.addDocument(document);
-
- document = new Document();
- document.add(storedField);
- Field termVectorField = newField("termVector", "termVector",
- Field.Store.NO, Field.Index.NOT_ANALYZED,
- Field.TermVector.WITH_POSITIONS_OFFSETS);
- document.add(termVectorField);
- writer.addDocument(document);
- writer.optimize();
- writer.close();
-
- IndexReader reader = IndexReader.open(dir, true);
- assertTrue(reader.getTermFreqVectors(0)==null);
- assertTrue(reader.getTermFreqVectors(1)==null);
- assertTrue(reader.getTermFreqVectors(2)!=null);
- reader.close();
- }
- dir.close();
- }
-
- // LUCENE-1168
- public void testTermVectorCorruption3() throws IOException {
- Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMaxBufferedDocs(2).setRAMBufferSizeMB(
- IndexWriterConfig.DISABLE_AUTO_FLUSH).setMergeScheduler(
- new SerialMergeScheduler()).setMergePolicy(new LogDocMergePolicy()));
-
- Document document = new Document();
-
- document = new Document();
- Field storedField = newField("stored", "stored", Field.Store.YES,
- Field.Index.NO);
- document.add(storedField);
- Field termVectorField = newField("termVector", "termVector",
- Field.Store.NO, Field.Index.NOT_ANALYZED,
- Field.TermVector.WITH_POSITIONS_OFFSETS);
- document.add(termVectorField);
- for(int i=0;i<10;i++)
- writer.addDocument(document);
- writer.close();
-
- writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT,
- new MockAnalyzer(random)).setMaxBufferedDocs(2)
- .setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
- .setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(
- new LogDocMergePolicy()));
- for(int i=0;i<6;i++)
- writer.addDocument(document);
-
- writer.optimize();
- writer.close();
-
- IndexReader reader = IndexReader.open(dir, true);
- for(int i=0;i<10;i++) {
- reader.getTermFreqVectors(i);
- reader.document(i);
- }
- reader.close();
- dir.close();
- }
-
- // LUCENE-1008
- public void testNoTermVectorAfterTermVector() throws IOException {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document document = new Document();
- document.add(newField("tvtest", "a b c", Field.Store.NO, Field.Index.ANALYZED,
- Field.TermVector.YES));
- iw.addDocument(document);
- document = new Document();
- document.add(newField("tvtest", "x y z", Field.Store.NO, Field.Index.ANALYZED,
- Field.TermVector.NO));
- iw.addDocument(document);
- // Make first segment
- iw.commit();
-
- document.add(newField("tvtest", "a b c", Field.Store.NO, Field.Index.ANALYZED,
- Field.TermVector.YES));
- iw.addDocument(document);
- // Make 2nd segment
- iw.commit();
-
- iw.optimize();
- iw.close();
- dir.close();
- }
-
- // LUCENE-1010
- public void testNoTermVectorAfterTermVectorMerge() throws IOException {
- Directory dir = newDirectory();
- IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer(random)));
- Document document = new Document();
- document.add(newField("tvtest", "a b c", Field.Store.NO, Field.Index.ANALYZED,
- Field.TermVector.YES));
- iw.addDocument(document);
- iw.commit();
-
- document = new Document();
- document.add(newField("tvtest", "x y z", Field.Store.NO, Field.Index.ANALYZED,
- Field.TermVector.NO));
- iw.addDocument(document);
- // Make first segment
- iw.commit();
-
- iw.optimize();
-
- document.add(newField("tvtest", "a b c", Field.Store.NO, Field.Index.ANALYZED,
- Field.TermVector.YES));
- iw.addDocument(document);
- // Make 2nd segment
- iw.commit();
- iw.optimize();
-
- iw.close();
- dir.close();
- }
-}
fnp / pylucene.git / blobdiff