--- /dev/null
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Random;
+import java.util.Set;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.UnicodeUtil;
+
+public class TestIndexWriterUnicode extends LuceneTestCase {
+
+ final String[] utf8Data = new String[] {
+ // unpaired low surrogate
+ "ab\udc17cd", "ab\ufffdcd",
+ "\udc17abcd", "\ufffdabcd",
+ "\udc17", "\ufffd",
+ "ab\udc17\udc17cd", "ab\ufffd\ufffdcd",
+ "\udc17\udc17abcd", "\ufffd\ufffdabcd",
+ "\udc17\udc17", "\ufffd\ufffd",
+
+ // unpaired high surrogate
+ "ab\ud917cd", "ab\ufffdcd",
+ "\ud917abcd", "\ufffdabcd",
+ "\ud917", "\ufffd",
+ "ab\ud917\ud917cd", "ab\ufffd\ufffdcd",
+ "\ud917\ud917abcd", "\ufffd\ufffdabcd",
+ "\ud917\ud917", "\ufffd\ufffd",
+
+ // backwards surrogates
+ "ab\udc17\ud917cd", "ab\ufffd\ufffdcd",
+ "\udc17\ud917abcd", "\ufffd\ufffdabcd",
+ "\udc17\ud917", "\ufffd\ufffd",
+ "ab\udc17\ud917\udc17\ud917cd", "ab\ufffd\ud917\udc17\ufffdcd",
+ "\udc17\ud917\udc17\ud917abcd", "\ufffd\ud917\udc17\ufffdabcd",
+ "\udc17\ud917\udc17\ud917", "\ufffd\ud917\udc17\ufffd"
+ };
+
+ private int nextInt(int lim) {
+ return random.nextInt(lim);
+ }
+
+ private int nextInt(int start, int end) {
+ return start + nextInt(end-start);
+ }
+
+ private boolean fillUnicode(char[] buffer, char[] expected, int offset, int count) {
+ final int len = offset + count;
+ boolean hasIllegal = false;
+
+ if (offset > 0 && buffer[offset] >= 0xdc00 && buffer[offset] < 0xe000)
+ // Don't start in the middle of a valid surrogate pair
+ offset--;
+
+ for(int i=offset;i<len;i++) {
+ int t = nextInt(6);
+ if (0 == t && i < len-1) {
+ // Make a surrogate pair
+ // High surrogate
+ expected[i] = buffer[i++] = (char) nextInt(0xd800, 0xdc00);
+ // Low surrogate
+ expected[i] = buffer[i] = (char) nextInt(0xdc00, 0xe000);
+ } else if (t <= 1)
+ expected[i] = buffer[i] = (char) nextInt(0x80);
+ else if (2 == t)
+ expected[i] = buffer[i] = (char) nextInt(0x80, 0x800);
+ else if (3 == t)
+ expected[i] = buffer[i] = (char) nextInt(0x800, 0xd800);
+ else if (4 == t)
+ expected[i] = buffer[i] = (char) nextInt(0xe000, 0xffff);
+ else if (5 == t && i < len-1) {
+ // Illegal unpaired surrogate
+ if (nextInt(10) == 7) {
+ if (random.nextBoolean())
+ buffer[i] = (char) nextInt(0xd800, 0xdc00);
+ else
+ buffer[i] = (char) nextInt(0xdc00, 0xe000);
+ expected[i++] = 0xfffd;
+ expected[i] = buffer[i] = (char) nextInt(0x800, 0xd800);
+ hasIllegal = true;
+ } else
+ expected[i] = buffer[i] = (char) nextInt(0x800, 0xd800);
+ } else {
+ expected[i] = buffer[i] = ' ';
+ }
+ }
+
+ return hasIllegal;
+ }
+
+ // both start & end are inclusive
+ private final int getInt(Random r, int start, int end) {
+ return start + r.nextInt(1+end-start);
+ }
+
+ private final String asUnicodeChar(char c) {
+ return "U+" + Integer.toHexString(c);
+ }
+
+ private final String termDesc(String s) {
+ final String s0;
+ assertTrue(s.length() <= 2);
+ if (s.length() == 1) {
+ s0 = asUnicodeChar(s.charAt(0));
+ } else {
+ s0 = asUnicodeChar(s.charAt(0)) + "," + asUnicodeChar(s.charAt(1));
+ }
+ return s0;
+ }
+
+ // LUCENE-510
+ public void testRandomUnicodeStrings() throws Throwable {
+ char[] buffer = new char[20];
+ char[] expected = new char[20];
+
+ UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result();
+ UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
+
+ int num = atLeast(100000);
+ for (int iter = 0; iter < num; iter++) {
+ boolean hasIllegal = fillUnicode(buffer, expected, 0, 20);
+
+ UnicodeUtil.UTF16toUTF8(buffer, 0, 20, utf8);
+ if (!hasIllegal) {
+ byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
+ assertEquals(b.length, utf8.length);
+ for(int i=0;i<b.length;i++)
+ assertEquals(b[i], utf8.result[i]);
+ }
+
+ UnicodeUtil.UTF8toUTF16(utf8.result, 0, utf8.length, utf16);
+ assertEquals(utf16.length, 20);
+ for(int i=0;i<20;i++)
+ assertEquals(expected[i], utf16.result[i]);
+ }
+ }
+
+ // LUCENE-510
+ public void testAllUnicodeChars() throws Throwable {
+
+ UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result();
+ UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
+ char[] chars = new char[2];
+ for(int ch=0;ch<0x0010FFFF;ch++) {
+
+ if (ch == 0xd800)
+ // Skip invalid code points
+ ch = 0xe000;
+
+ int len = 0;
+ if (ch <= 0xffff) {
+ chars[len++] = (char) ch;
+ } else {
+ chars[len++] = (char) (((ch-0x0010000) >> 10) + UnicodeUtil.UNI_SUR_HIGH_START);
+ chars[len++] = (char) (((ch-0x0010000) & 0x3FFL) + UnicodeUtil.UNI_SUR_LOW_START);
+ }
+
+ UnicodeUtil.UTF16toUTF8(chars, 0, len, utf8);
+
+ String s1 = new String(chars, 0, len);
+ String s2 = new String(utf8.result, 0, utf8.length, "UTF-8");
+ assertEquals("codepoint " + ch, s1, s2);
+
+ UnicodeUtil.UTF8toUTF16(utf8.result, 0, utf8.length, utf16);
+ assertEquals("codepoint " + ch, s1, new String(utf16.result, 0, utf16.length));
+
+ byte[] b = s1.getBytes("UTF-8");
+ assertEquals(utf8.length, b.length);
+ for(int j=0;j<utf8.length;j++)
+ assertEquals(utf8.result[j], b[j]);
+ }
+ }
+
+ public void testEmbeddedFFFF() throws Throwable {
+
+ Directory d = newDirectory();
+ IndexWriter w = new IndexWriter(d, newIndexWriterConfig( TEST_VERSION_CURRENT, new TestIndexWriter.StringSplitAnalyzer()));
+ Document doc = new Document();
+ doc.add(newField("field", "a a\uffffb", Field.Store.NO, Field.Index.ANALYZED));
+ w.addDocument(doc);
+ doc = new Document();
+ doc.add(newField("field", "a", Field.Store.NO, Field.Index.ANALYZED));
+ w.addDocument(doc);
+ w.close();
+
+ d.close();
+ }
+
+ // LUCENE-510
+ public void testInvalidUTF16() throws Throwable {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new TestIndexWriter.StringSplitAnalyzer()));
+ Document doc = new Document();
+
+ final int count = utf8Data.length/2;
+ for(int i=0;i<count;i++)
+ doc.add(newField("f" + i, utf8Data[2*i], Field.Store.YES, Field.Index.ANALYZED));
+ w.addDocument(doc);
+ w.close();
+
+ IndexReader ir = IndexReader.open(dir, true);
+ Document doc2 = ir.document(0);
+ for(int i=0;i<count;i++) {
+ assertEquals("field " + i + " was not indexed correctly", 1, ir.docFreq(new Term("f"+i, utf8Data[2*i+1])));
+ assertEquals("field " + i + " is incorrect", utf8Data[2*i+1], doc2.getField("f"+i).stringValue());
+ }
+ ir.close();
+ dir.close();
+ }
+
+ // LUCENE-510
+ public void testIncrementalUnicodeStrings() throws Throwable {
+ char[] buffer = new char[20];
+ char[] expected = new char[20];
+
+ UnicodeUtil.UTF8Result utf8 = new UnicodeUtil.UTF8Result();
+ UnicodeUtil.UTF16Result utf16 = new UnicodeUtil.UTF16Result();
+ UnicodeUtil.UTF16Result utf16a = new UnicodeUtil.UTF16Result();
+
+ boolean hasIllegal = false;
+ byte[] last = new byte[60];
+
+ int num = atLeast(100000);
+ for (int iter = 0; iter < num; iter++) {
+
+ final int prefix;
+
+ if (iter == 0 || hasIllegal)
+ prefix = 0;
+ else
+ prefix = nextInt(20);
+
+ hasIllegal = fillUnicode(buffer, expected, prefix, 20-prefix);
+
+ UnicodeUtil.UTF16toUTF8(buffer, 0, 20, utf8);
+ if (!hasIllegal) {
+ byte[] b = new String(buffer, 0, 20).getBytes("UTF-8");
+ assertEquals(b.length, utf8.length);
+ for(int i=0;i<b.length;i++)
+ assertEquals(b[i], utf8.result[i]);
+ }
+
+ int bytePrefix = 20;
+ if (iter == 0 || hasIllegal)
+ bytePrefix = 0;
+ else
+ for(int i=0;i<20;i++)
+ if (last[i] != utf8.result[i]) {
+ bytePrefix = i;
+ break;
+ }
+ System.arraycopy(utf8.result, 0, last, 0, utf8.length);
+
+ UnicodeUtil.UTF8toUTF16(utf8.result, bytePrefix, utf8.length-bytePrefix, utf16);
+ assertEquals(20, utf16.length);
+ for(int i=0;i<20;i++)
+ assertEquals(expected[i], utf16.result[i]);
+
+ UnicodeUtil.UTF8toUTF16(utf8.result, 0, utf8.length, utf16a);
+ assertEquals(20, utf16a.length);
+ for(int i=0;i<20;i++)
+ assertEquals(expected[i], utf16a.result[i]);
+ }
+ }
+}
fnp / pylucene.git / blobdiff