+++ /dev/null
-package org.apache.lucene.index;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.File;
-import java.io.IOException;
-import java.io.ByteArrayInputStream;
-import java.io.DataInputStream;
-import java.util.Arrays;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Random;
-
-import org.apache.lucene.analysis.WhitespaceAnalyzer;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.FieldSelector;
-import org.apache.lucene.document.FieldSelectorResult;
-import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.document.NumericField;
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.ScoreDoc;
-import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.NumericRangeQuery;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.ReaderUtil;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util._TestUtil;
-import org.apache.lucene.util.Constants;
-
-/*
- Verify we can read the pre-2.1 file format, do searches
- against it, and add documents to it.
-*/
-
-public class TestBackwardsCompatibility extends LuceneTestCase {
-
- // Uncomment these cases & run them on an older Lucene
- // version, to generate an index to test backwards
- // compatibility. Then, cd to build/test/index.cfs and
- // run "zip index.<VERSION>.cfs.zip *"; cd to
- // build/test/index.nocfs and run "zip
- // index.<VERSION>.nocfs.zip *". Then move those 2 zip
- // files to your trunk checkout and add them to the
- // oldNames array.
-
- /*
- public void testCreateCFS() throws IOException {
- createIndex("index.cfs", true, false);
- }
-
- public void testCreateNoCFS() throws IOException {
- createIndex("index.nocfs", false, false);
- }
- */
-
- /*
- // These are only needed for the special upgrade test to verify
- // that also optimized indexes are correctly upgraded by IndexUpgrader.
- // You don't need them to be build for non-3.1 (the test is happy with just one
- // "old" segment format, version is unimportant:
-
- public void testCreateOptimizedCFS() throws IOException {
- createIndex("index.optimized.cfs", true, true);
- }
-
- public void testCreateOptimizedNoCFS() throws IOException {
- createIndex("index.optimized.nocfs", false, true);
- }
- */
-
- final String[] oldNames = {"19.cfs",
- "19.nocfs",
- "20.cfs",
- "20.nocfs",
- "21.cfs",
- "21.nocfs",
- "22.cfs",
- "22.nocfs",
- "23.cfs",
- "23.nocfs",
- "24.cfs",
- "24.nocfs",
- "29.cfs",
- "29.nocfs",
- "30.cfs",
- "30.nocfs",
- "31.cfs",
- "31.nocfs",
- };
-
- final String[] oldOptimizedNames = {"31.optimized.cfs",
- "31.optimized.nocfs",
- };
-
- private void assertCompressedFields29(Directory dir, boolean shouldStillBeCompressed) throws IOException {
- int count = 0;
- final int TEXT_PLAIN_LENGTH = TEXT_TO_COMPRESS.length() * 2;
- // FieldSelectorResult.SIZE returns 2*number_of_chars for String fields:
- final int BINARY_PLAIN_LENGTH = BINARY_TO_COMPRESS.length;
-
- IndexReader reader = IndexReader.open(dir, true);
- try {
- // look into sub readers and check if raw merge is on/off
- List<IndexReader> readers = new ArrayList<IndexReader>();
- ReaderUtil.gatherSubReaders(readers, reader);
- for (IndexReader ir : readers) {
- final FieldsReader fr = ((SegmentReader) ir).getFieldsReader();
- assertTrue("for a 2.9 index, FieldsReader.canReadRawDocs() must be false and other way round for a trunk index",
- shouldStillBeCompressed != fr.canReadRawDocs());
- }
-
- // test that decompression works correctly
- for(int i=0; i<reader.maxDoc(); i++) {
- if (!reader.isDeleted(i)) {
- Document d = reader.document(i);
- if (d.get("content3") != null) continue;
- count++;
- Fieldable compressed = d.getFieldable("compressed");
- if (Integer.parseInt(d.get("id")) % 2 == 0) {
- assertFalse(compressed.isBinary());
- assertEquals("incorrectly decompressed string", TEXT_TO_COMPRESS, compressed.stringValue());
- } else {
- assertTrue(compressed.isBinary());
- assertTrue("incorrectly decompressed binary", Arrays.equals(BINARY_TO_COMPRESS, compressed.getBinaryValue()));
- }
- }
- }
-
- // check if field was decompressed after optimize
- for(int i=0; i<reader.maxDoc(); i++) {
- if (!reader.isDeleted(i)) {
- Document d = reader.document(i, new FieldSelector() {
- public FieldSelectorResult accept(String fieldName) {
- return ("compressed".equals(fieldName)) ? FieldSelectorResult.SIZE : FieldSelectorResult.LOAD;
- }
- });
- if (d.get("content3") != null) continue;
- count++;
- // read the size from the binary value using DataInputStream (this prevents us from doing the shift ops ourselves):
- final DataInputStream ds = new DataInputStream(new ByteArrayInputStream(d.getFieldable("compressed").getBinaryValue()));
- final int actualSize = ds.readInt();
- ds.close();
- final int compressedSize = Integer.parseInt(d.get("compressedSize"));
- final boolean binary = Integer.parseInt(d.get("id")) % 2 > 0;
- final int shouldSize = shouldStillBeCompressed ?
- compressedSize :
- (binary ? BINARY_PLAIN_LENGTH : TEXT_PLAIN_LENGTH);
- assertEquals("size incorrect", shouldSize, actualSize);
- if (!shouldStillBeCompressed) {
- assertFalse("uncompressed field should have another size than recorded in index", compressedSize == actualSize);
- }
- }
- }
- assertEquals("correct number of tests", 34 * 2, count);
- } finally {
- reader.close();
- }
- }
-
- public void testUpgrade29Compression() throws IOException {
- int hasTested29 = 0;
-
- for(int i=0;i<oldNames.length;i++) {
- File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
- _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
- Directory dir = newFSDirectory(oldIndxeDir);
-
- if (oldNames[i].startsWith("29.")) {
- assertCompressedFields29(dir, true);
- hasTested29++;
- }
-
- new IndexUpgrader(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null), VERBOSE ? System.out : null, false)
- .upgrade();
-
- if (oldNames[i].startsWith("29.")) {
- assertCompressedFields29(dir, false);
- hasTested29++;
- }
-
- dir.close();
- _TestUtil.rmDir(oldIndxeDir);
- }
-
- assertEquals("test for compressed field should have run 4 times", 4, hasTested29);
- }
-
- public void testAddOldIndexes() throws IOException {
- for (String name : oldNames) {
- File oldIndxeDir = _TestUtil.getTempDir(name);
- _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
- Directory dir = newFSDirectory(oldIndxeDir);
-
- Directory targetDir = newDirectory();
- IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
- w.addIndexes(new Directory[] { dir });
- w.close();
-
- dir.close();
- targetDir.close();
- _TestUtil.rmDir(oldIndxeDir);
- }
- }
-
- public void testAddOldIndexesReader() throws IOException {
- for (String name : oldNames) {
- File oldIndxeDir = _TestUtil.getTempDir(name);
- _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
- Directory dir = newFSDirectory(oldIndxeDir);
- IndexReader reader = IndexReader.open(dir);
-
- Directory targetDir = newDirectory();
- IndexWriter w = new IndexWriter(targetDir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
- w.addIndexes(new IndexReader[] { reader });
- w.close();
- reader.close();
-
- dir.close();
- targetDir.close();
- _TestUtil.rmDir(oldIndxeDir);
- }
- }
-
- public void testSearchOldIndex() throws IOException {
- for(int i=0;i<oldNames.length;i++) {
- File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
- _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
- searchIndex(oldIndxeDir, oldNames[i]);
- _TestUtil.rmDir(oldIndxeDir);
- }
- }
-
- public void testIndexOldIndexNoAdds() throws IOException {
- for(int i=0;i<oldNames.length;i++) {
- File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
- _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
- changeIndexNoAdds(random, oldIndxeDir);
- _TestUtil.rmDir(oldIndxeDir);
- }
- }
-
- public void testIndexOldIndex() throws IOException {
- for(int i=0;i<oldNames.length;i++) {
- if (VERBOSE) {
- System.out.println("TEST: oldName=" + oldNames[i]);
- }
- File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]);
- _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir);
- changeIndexWithAdds(random, oldIndxeDir, oldNames[i]);
- _TestUtil.rmDir(oldIndxeDir);
- }
- }
-
- private void testHits(ScoreDoc[] hits, int expectedCount, IndexReader reader) throws IOException {
- final int hitCount = hits.length;
- assertEquals("wrong number of hits", expectedCount, hitCount);
- for(int i=0;i<hitCount;i++) {
- reader.document(hits[i].doc);
- reader.getTermFreqVectors(hits[i].doc);
- }
- }
-
- public void searchIndex(File indexDir, String oldName) throws IOException {
- //QueryParser parser = new QueryParser("contents", new WhitespaceAnalyzer(TEST_VERSION_CURRENT));
- //Query query = parser.parse("handle:1");
-
- Directory dir = newFSDirectory(indexDir);
- IndexSearcher searcher = new IndexSearcher(dir, true);
- IndexReader reader = searcher.getIndexReader();
-
- _TestUtil.checkIndex(dir);
-
- for(int i=0;i<35;i++) {
- if (!reader.isDeleted(i)) {
- Document d = reader.document(i);
- List<Fieldable> fields = d.getFields();
- if (!oldName.startsWith("19.") &&
- !oldName.startsWith("20.") &&
- !oldName.startsWith("21.") &&
- !oldName.startsWith("22.")) {
-
- if (d.getField("content3") == null) {
- final int numFields = oldName.startsWith("29.") ? 7 : 5;
- assertEquals(numFields, fields.size());
- Field f = d.getField("id");
- assertEquals(""+i, f.stringValue());
-
- f = d.getField("utf8");
- assertEquals("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());
-
- f = d.getField("autf8");
- assertEquals("Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", f.stringValue());
-
- f = d.getField("content2");
- assertEquals("here is more content with aaa aaa aaa", f.stringValue());
-
- f = d.getField("fie\u2C77ld");
- assertEquals("field with non-ascii name", f.stringValue());
- }
-
- TermFreqVector tfv = reader.getTermFreqVector(i, "utf8");
- assertNotNull("docID=" + i + " index=" + indexDir.getName(), tfv);
- assertTrue(tfv instanceof TermPositionVector);
- }
- } else
- // Only ID 7 is deleted
- assertEquals(7, i);
- }
-
- ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
-
- // First document should be #21 since it's norm was
- // increased:
- Document d = searcher.doc(hits[0].doc);
- assertEquals("didn't get the right document first", "21", d.get("id"));
-
- testHits(hits, 34, searcher.getIndexReader());
-
- if (!oldName.startsWith("19.") &&
- !oldName.startsWith("20.") &&
- !oldName.startsWith("21.") &&
- !oldName.startsWith("22.")) {
- // Test on indices >= 2.3
- hits = searcher.search(new TermQuery(new Term("utf8", "\u0000")), null, 1000).scoreDocs;
- assertEquals(34, hits.length);
- hits = searcher.search(new TermQuery(new Term("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne")), null, 1000).scoreDocs;
- assertEquals(34, hits.length);
- hits = searcher.search(new TermQuery(new Term("utf8", "ab\ud917\udc17cd")), null, 1000).scoreDocs;
- assertEquals(34, hits.length);
- }
-
- searcher.close();
- dir.close();
- }
-
- private int compare(String name, String v) {
- int v0 = Integer.parseInt(name.substring(0, 2));
- int v1 = Integer.parseInt(v);
- return v0 - v1;
- }
-
- /* Open pre-lockless index, add docs, do a delete &
- * setNorm, and search */
- public void changeIndexWithAdds(Random random, File oldIndexDir, String origOldName) throws IOException {
- Directory dir = newFSDirectory(oldIndexDir);
- // open writer
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND));
- writer.setInfoStream(VERBOSE ? System.out : null);
- // add 10 docs
- for(int i=0;i<10;i++) {
- addDoc(writer, 35+i);
- }
-
- // make sure writer sees right total -- writer seems not to know about deletes in .del?
- final int expected;
- if (compare(origOldName, "24") < 0) {
- expected = 44;
- } else {
- expected = 45;
- }
- assertEquals("wrong doc count", expected, writer.numDocs());
- writer.close();
-
- // make sure searching sees right # hits
- IndexSearcher searcher = new IndexSearcher(dir, true);
- ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
- Document d = searcher.doc(hits[0].doc);
- assertEquals("wrong first document", "21", d.get("id"));
- testHits(hits, 44, searcher.getIndexReader());
- searcher.close();
-
- // make sure we can do delete & setNorm against this
- // pre-lockless segment:
- IndexReader reader = IndexReader.open(dir, false);
- searcher = newSearcher(reader);
- Term searchTerm = new Term("id", "6");
- int delCount = reader.deleteDocuments(searchTerm);
- assertEquals("wrong delete count", 1, delCount);
- reader.setNorm(searcher.search(new TermQuery(new Term("id", "22")), 10).scoreDocs[0].doc, "content", (float) 2.0);
- reader.close();
- searcher.close();
-
- // make sure they "took":
- searcher = new IndexSearcher(dir, true);
- hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
- assertEquals("wrong number of hits", 43, hits.length);
- d = searcher.doc(hits[0].doc);
- assertEquals("wrong first document", "22", d.get("id"));
- testHits(hits, 43, searcher.getIndexReader());
- searcher.close();
-
- // optimize
- writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND));
- writer.optimize();
- writer.close();
-
- searcher = new IndexSearcher(dir, true);
- hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
- assertEquals("wrong number of hits", 43, hits.length);
- d = searcher.doc(hits[0].doc);
- testHits(hits, 43, searcher.getIndexReader());
- assertEquals("wrong first document", "22", d.get("id"));
- searcher.close();
-
- dir.close();
- }
-
- /* Open pre-lockless index, add docs, do a delete &
- * setNorm, and search */
- public void changeIndexNoAdds(Random random, File oldIndexDir) throws IOException {
-
- Directory dir = newFSDirectory(oldIndexDir);
-
- // make sure searching sees right # hits
- IndexSearcher searcher = new IndexSearcher(dir, true);
- ScoreDoc[] hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
- assertEquals("wrong number of hits", 34, hits.length);
- Document d = searcher.doc(hits[0].doc);
- assertEquals("wrong first document", "21", d.get("id"));
- searcher.close();
-
- // make sure we can do a delete & setNorm against this
- // pre-lockless segment:
- IndexReader reader = IndexReader.open(dir, false);
- Term searchTerm = new Term("id", "6");
- int delCount = reader.deleteDocuments(searchTerm);
- assertEquals("wrong delete count", 1, delCount);
- reader.setNorm(22, "content", (float) 2.0);
- reader.close();
-
- // make sure they "took":
- searcher = new IndexSearcher(dir, true);
- hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
- assertEquals("wrong number of hits", 33, hits.length);
- d = searcher.doc(hits[0].doc);
- assertEquals("wrong first document", "22", d.get("id"));
- testHits(hits, 33, searcher.getIndexReader());
- searcher.close();
-
- // optimize
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND));
- writer.optimize();
- writer.close();
-
- searcher = new IndexSearcher(dir, true);
- hits = searcher.search(new TermQuery(new Term("content", "aaa")), null, 1000).scoreDocs;
- assertEquals("wrong number of hits", 33, hits.length);
- d = searcher.doc(hits[0].doc);
- assertEquals("wrong first document", "22", d.get("id"));
- testHits(hits, 33, searcher.getIndexReader());
- searcher.close();
-
- dir.close();
- }
-
- public File createIndex(String dirName, boolean doCFS, boolean optimized) throws IOException {
- // we use a real directory name that is not cleaned up, because this method is only used to create backwards indexes:
- File indexDir = new File(LuceneTestCase.TEMP_DIR, dirName);
- _TestUtil.rmDir(indexDir);
- Directory dir = newFSDirectory(indexDir);
- IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxBufferedDocs(10);
- ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
- if (doCFS) {
- ((LogMergePolicy) conf.getMergePolicy()).setNoCFSRatio(1.0);
- }
- IndexWriter writer = new IndexWriter(dir, conf);
-
- for(int i=0;i<35;i++) {
- addDoc(writer, i);
- }
- assertEquals("wrong doc count", 35, writer.maxDoc());
- if (optimized) {
- writer.optimize();
- }
- writer.close();
-
- if (!optimized) {
- // open fresh writer so we get no prx file in the added segment
- conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxBufferedDocs(10);
- ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(doCFS);
- writer = new IndexWriter(dir, conf);
- addNoProxDoc(writer);
- writer.close();
-
- // Delete one doc so we get a .del file:
- IndexReader reader = IndexReader.open(dir, false);
- Term searchTerm = new Term("id", "7");
- int delCount = reader.deleteDocuments(searchTerm);
- assertEquals("didn't delete the right number of documents", 1, delCount);
-
- // Set one norm so we get a .s0 file:
- reader.setNorm(21, "content", (float) 1.5);
- reader.close();
- }
-
- dir.close();
-
- return indexDir;
- }
-
- /* Verifies that the expected file names were produced */
-
- public void testExactFileNames() throws IOException {
-
- String outputDirName = "lucene.backwardscompat0.index";
- File outputDir = _TestUtil.getTempDir(outputDirName);
- _TestUtil.rmDir(outputDir);
-
- try {
- Directory dir = newFSDirectory(outputDir);
-
- LogMergePolicy mergePolicy = newLogMergePolicy(true, 10);
- mergePolicy.setNoCFSRatio(1); // This test expects all of its segments to be in CFS
- IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxBufferedDocs(-1).setRAMBufferSizeMB(16.0)
- .setMergePolicy(mergePolicy);
- IndexWriter writer = new IndexWriter(dir, conf);
- for(int i=0;i<35;i++) {
- addDoc(writer, i);
- }
- assertEquals("wrong doc count", 35, writer.maxDoc());
- writer.close();
-
- // Delete one doc so we get a .del file:
- IndexReader reader = IndexReader.open(dir, false);
- Term searchTerm = new Term("id", "7");
- int delCount = reader.deleteDocuments(searchTerm);
- assertEquals("didn't delete the right number of documents", 1, delCount);
-
- // Set one norm so we get a .s0 file:
- reader.setNorm(21, "content", (float) 1.5);
- reader.close();
-
- // The numbering of fields can vary depending on which
- // JRE is in use. On some JREs we see content bound to
- // field 0; on others, field 1. So, here we have to
- // figure out which field number corresponds to
- // "content", and then set our expected file names below
- // accordingly:
- CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs");
- FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm");
- int contentFieldIndex = -1;
- for(int i=0;i<fieldInfos.size();i++) {
- FieldInfo fi = fieldInfos.fieldInfo(i);
- if (fi.name.equals("content")) {
- contentFieldIndex = i;
- break;
- }
- }
- cfsReader.close();
- assertTrue("could not locate the 'content' field number in the _2.cfs segment", contentFieldIndex != -1);
-
- // Now verify file names:
- String[] expected = new String[] {"_0.cfs",
- "_0_1.del",
- "_0_1.s" + contentFieldIndex,
- "segments_2",
- "segments.gen"};
-
- String[] actual = dir.listAll();
- Arrays.sort(expected);
- Arrays.sort(actual);
- if (!Arrays.equals(expected, actual)) {
- fail("incorrect filenames in index: expected:\n " + asString(expected) + "\n actual:\n " + asString(actual));
- }
- dir.close();
- } finally {
- _TestUtil.rmDir(outputDir);
- }
- }
-
- private String asString(String[] l) {
- String s = "";
- for(int i=0;i<l.length;i++) {
- if (i > 0) {
- s += "\n ";
- }
- s += l[i];
- }
- return s;
- }
-
- private void addDoc(IndexWriter writer, int id) throws IOException
- {
- Document doc = new Document();
- doc.add(new Field("content", "aaa", Field.Store.NO, Field.Index.ANALYZED));
- doc.add(new Field("id", Integer.toString(id), Field.Store.YES, Field.Index.NOT_ANALYZED));
- doc.add(new Field("autf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- doc.add(new Field("utf8", "Lu\uD834\uDD1Ece\uD834\uDD60ne \u0000 \u2620 ab\ud917\udc17cd", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- doc.add(new Field("content2", "here is more content with aaa aaa aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- doc.add(new Field("fie\u2C77ld", "field with non-ascii name", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
- /* This was used in 2.9 to generate an index with compressed field:
- if (id % 2 == 0) {
- doc.add(new Field("compressed", TEXT_TO_COMPRESS, Field.Store.COMPRESS, Field.Index.NOT_ANALYZED));
- doc.add(new Field("compressedSize", Integer.toString(TEXT_COMPRESSED_LENGTH), Field.Store.YES, Field.Index.NOT_ANALYZED));
- } else {
- doc.add(new Field("compressed", BINARY_TO_COMPRESS, Field.Store.COMPRESS));
- doc.add(new Field("compressedSize", Integer.toString(BINARY_COMPRESSED_LENGTH), Field.Store.YES, Field.Index.NOT_ANALYZED));
- }
- */
- // add numeric fields, to test if later versions preserve encoding
- doc.add(new NumericField("trieInt", 4).setIntValue(id));
- doc.add(new NumericField("trieLong", 4).setLongValue(id));
- writer.addDocument(doc);
- }
-
- private void addNoProxDoc(IndexWriter writer) throws IOException {
- Document doc = new Document();
- Field f = new Field("content3", "aaa", Field.Store.YES, Field.Index.ANALYZED);
- f.setOmitTermFreqAndPositions(true);
- doc.add(f);
- f = new Field("content4", "aaa", Field.Store.YES, Field.Index.NO);
- f.setOmitTermFreqAndPositions(true);
- doc.add(f);
- writer.addDocument(doc);
- }
-
- static final String TEXT_TO_COMPRESS = "this is a compressed field and should appear in 3.0 as an uncompressed field after merge";
- // FieldSelectorResult.SIZE returns compressed size for compressed fields, which are internally handled as binary;
- // do it in the same way like FieldsWriter, do not use CompressionTools.compressString() for compressed fields:
- /* This was used in 2.9 to generate an index with compressed field:
- static final int TEXT_COMPRESSED_LENGTH;
- static {
- try {
- TEXT_COMPRESSED_LENGTH = CompressionTools.compress(TEXT_TO_COMPRESS.getBytes("UTF-8")).length;
- } catch (Exception e) {
- throw new RuntimeException();
- }
- }
- */
- static final byte[] BINARY_TO_COMPRESS = new byte[]{1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20};
- /* This was used in 2.9 to generate an index with compressed field:
- static final int BINARY_COMPRESSED_LENGTH = CompressionTools.compress(BINARY_TO_COMPRESS).length;
- */
-
- public void testNumericFields() throws Exception {
- for(int i=0;i<oldNames.length;i++) {
- // only test indexes >= 3.0
- if (oldNames[i].compareTo("30.") < 0) continue;
-
- File oldIndexDir = _TestUtil.getTempDir(oldNames[i]);
- _TestUtil.unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndexDir);
- Directory dir = newFSDirectory(oldIndexDir);
- IndexSearcher searcher = new IndexSearcher(dir, true);
-
- for (int id=10; id<15; id++) {
- ScoreDoc[] hits = searcher.search(NumericRangeQuery.newIntRange("trieInt", 4, Integer.valueOf(id), Integer.valueOf(id), true, true), 100).scoreDocs;
- assertEquals("wrong number of hits", 1, hits.length);
- Document d = searcher.doc(hits[0].doc);
- assertEquals(String.valueOf(id), d.get("id"));
-
- hits = searcher.search(NumericRangeQuery.newLongRange("trieLong", 4, Long.valueOf(id), Long.valueOf(id), true, true), 100).scoreDocs;
- assertEquals("wrong number of hits", 1, hits.length);
- d = searcher.doc(hits[0].doc);
- assertEquals(String.valueOf(id), d.get("id"));
- }
-
- // check that also lower-precision fields are ok
- ScoreDoc[] hits = searcher.search(NumericRangeQuery.newIntRange("trieInt", 4, Integer.MIN_VALUE, Integer.MAX_VALUE, false, false), 100).scoreDocs;
- assertEquals("wrong number of hits", 34, hits.length);
-
- hits = searcher.search(NumericRangeQuery.newLongRange("trieLong", 4, Long.MIN_VALUE, Long.MAX_VALUE, false, false), 100).scoreDocs;
- assertEquals("wrong number of hits", 34, hits.length);
-
- // check decoding into field cache
- int[] fci = FieldCache.DEFAULT.getInts(searcher.getIndexReader(), "trieInt");
- for (int val : fci) {
- assertTrue("value in id bounds", val >= 0 && val < 35);
- }
-
- long[] fcl = FieldCache.DEFAULT.getLongs(searcher.getIndexReader(), "trieLong");
- for (long val : fcl) {
- assertTrue("value in id bounds", val >= 0L && val < 35L);
- }
-
- searcher.close();
- dir.close();
- _TestUtil.rmDir(oldIndexDir);
- }
- }
-
- private int checkAllSegmentsUpgraded(Directory dir) throws IOException {
- final SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- if (VERBOSE) {
- System.out.println("checkAllSegmentsUpgraded: " + infos);
- }
- for (SegmentInfo si : infos) {
- assertEquals(Constants.LUCENE_MAIN_VERSION, si.getVersion());
- }
- return infos.size();
- }
-
- private int getNumberOfSegments(Directory dir) throws IOException {
- final SegmentInfos infos = new SegmentInfos();
- infos.read(dir);
- return infos.size();
- }
-
- public void testUpgradeOldIndex() throws Exception {
- List<String> names = new ArrayList<String>(oldNames.length + oldOptimizedNames.length);
- names.addAll(Arrays.asList(oldNames));
- names.addAll(Arrays.asList(oldOptimizedNames));
- for(String name : names) {
- if (VERBOSE) {
- System.out.println("testUpgradeOldIndex: index=" +name);
- }
- File oldIndxeDir = _TestUtil.getTempDir(name);
- _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
- Directory dir = newFSDirectory(oldIndxeDir);
-
- new IndexUpgrader(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null), VERBOSE ? System.out : null, false)
- .upgrade();
-
- checkAllSegmentsUpgraded(dir);
-
- dir.close();
- _TestUtil.rmDir(oldIndxeDir);
- }
- }
-
- public void testUpgradeOldOptimizedIndexWithAdditions() throws Exception {
- for (String name : oldOptimizedNames) {
- if (VERBOSE) {
- System.out.println("testUpgradeOldOptimizedIndexWithAdditions: index=" +name);
- }
- File oldIndxeDir = _TestUtil.getTempDir(name);
- _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir);
- Directory dir = newFSDirectory(oldIndxeDir);
-
- assertEquals("Original index must be optimized", 1, getNumberOfSegments(dir));
-
- // create a bunch of dummy segments
- int id = 40;
- RAMDirectory ramDir = new RAMDirectory();
- for (int i = 0; i < 3; i++) {
- // only use Log- or TieredMergePolicy, to make document addition predictable and not suddenly merge:
- MergePolicy mp = random.nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
- IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))
- .setMergePolicy(mp);
- IndexWriter w = new IndexWriter(ramDir, iwc);
- // add few more docs:
- for(int j = 0; j < RANDOM_MULTIPLIER * random.nextInt(30); j++) {
- addDoc(w, id++);
- }
- w.close(false);
- }
-
- // add dummy segments (which are all in current version) to optimized index
- MergePolicy mp = random.nextBoolean() ? newLogMergePolicy() : newTieredMergePolicy();
- IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, null)
- .setMergePolicy(mp);
- IndexWriter w = new IndexWriter(dir, iwc);
- w.setInfoStream(VERBOSE ? System.out : null);
- w.addIndexes(ramDir);
- w.close(false);
-
- // determine count of segments in modified index
- final int origSegCount = getNumberOfSegments(dir);
-
- new IndexUpgrader(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, null), VERBOSE ? System.out : null, false)
- .upgrade();
-
- final int segCount = checkAllSegmentsUpgraded(dir);
- assertEquals("Index must still contain the same number of segments, as only one segment was upgraded and nothing else merged",
- origSegCount, segCount);
-
- dir.close();
- _TestUtil.rmDir(oldIndxeDir);
- }
- }
-
-}
fnp / pylucene.git / blobdiff