--- /dev/null
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+
+import org.apache.lucene.util.Version;
+
+/**
+ * Testcase for {@link CharTokenizer} subclasses
+ */
+public class TestCharTokenizers extends BaseTokenStreamTestCase {
+
+ /*
+ * test to read surrogate pairs without loosing the pairing
+ * if the surrogate pair is at the border of the internal IO buffer
+ */
+ public void testReadSupplementaryChars() throws IOException {
+ StringBuilder builder = new StringBuilder();
+ // create random input
+ int num = 1024 + random.nextInt(1024);
+ num *= RANDOM_MULTIPLIER;
+ for (int i = 1; i < num; i++) {
+ builder.append("\ud801\udc1cabc");
+ if((i % 10) == 0)
+ builder.append(" ");
+ }
+ // internal buffer size is 1024 make sure we have a surrogate pair right at the border
+ builder.insert(1023, "\ud801\udc1c");
+ LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(
+ TEST_VERSION_CURRENT, new StringReader(builder.toString()));
+ assertTokenStreamContents(tokenizer, builder.toString().toLowerCase().split(" "));
+ }
+
+ /*
+ * test to extend the buffer TermAttribute buffer internally. If the internal
+ * alg that extends the size of the char array only extends by 1 char and the
+ * next char to be filled in is a supplementary codepoint (using 2 chars) an
+ * index out of bound exception is triggered.
+ */
+ public void testExtendCharBuffer() throws IOException {
+ for (int i = 0; i < 40; i++) {
+ StringBuilder builder = new StringBuilder();
+ for (int j = 0; j < 1+i; j++) {
+ builder.append("a");
+ }
+ builder.append("\ud801\udc1cabc");
+ LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(
+ TEST_VERSION_CURRENT, new StringReader(builder.toString()));
+ assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase()});
+ }
+ }
+
+ /*
+ * tests the max word length of 255 - tokenizer will split at the 255 char no matter what happens
+ */
+ public void testMaxWordLength() throws IOException {
+ StringBuilder builder = new StringBuilder();
+
+ for (int i = 0; i < 255; i++) {
+ builder.append("A");
+ }
+ LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(
+ TEST_VERSION_CURRENT, new StringReader(builder.toString() + builder.toString()));
+ assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(), builder.toString().toLowerCase()});
+ }
+
+ /*
+ * tests the max word length of 255 with a surrogate pair at position 255
+ */
+ public void testMaxWordLengthWithSupplementary() throws IOException {
+ StringBuilder builder = new StringBuilder();
+
+ for (int i = 0; i < 254; i++) {
+ builder.append("A");
+ }
+ builder.append("\ud801\udc1c");
+ LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(
+ TEST_VERSION_CURRENT, new StringReader(builder.toString() + builder.toString()));
+ assertTokenStreamContents(tokenizer, new String[] {builder.toString().toLowerCase(), builder.toString().toLowerCase()});
+ }
+
+ public void testLowerCaseTokenizer() throws IOException {
+ StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
+ LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(TEST_VERSION_CURRENT,
+ reader);
+ assertTokenStreamContents(tokenizer, new String[] { "tokenizer",
+ "\ud801\udc44test" });
+ }
+
+ public void testLowerCaseTokenizerBWCompat() throws IOException {
+ StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
+ LowerCaseTokenizer tokenizer = new LowerCaseTokenizer(Version.LUCENE_30,
+ reader);
+ assertTokenStreamContents(tokenizer, new String[] { "tokenizer", "test" });
+ }
+
+ public void testWhitespaceTokenizer() throws IOException {
+ StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT,
+ reader);
+ assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
+ "\ud801\udc1ctest" });
+ }
+
+ public void testWhitespaceTokenizerBWCompat() throws IOException {
+ StringReader reader = new StringReader("Tokenizer \ud801\udc1ctest");
+ WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(Version.LUCENE_30,
+ reader);
+ assertTokenStreamContents(tokenizer, new String[] { "Tokenizer",
+ "\ud801\udc1ctest" });
+ }
+
+ public void testIsTokenCharCharInSubclass() {
+ new TestingCharTokenizer(Version.LUCENE_30, new StringReader(""));
+ try {
+ new TestingCharTokenizer(TEST_VERSION_CURRENT, new StringReader(""));
+ fail("version 3.1 is not permitted if char based method is implemented");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ public void testNormalizeCharInSubclass() {
+ new TestingCharTokenizerNormalize(Version.LUCENE_30, new StringReader(""));
+ try {
+ new TestingCharTokenizerNormalize(TEST_VERSION_CURRENT,
+ new StringReader(""));
+ fail("version 3.1 is not permitted if char based method is implemented");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ public void testNormalizeAndIsTokenCharCharInSubclass() {
+ new TestingCharTokenizerNormalizeIsTokenChar(Version.LUCENE_30,
+ new StringReader(""));
+ try {
+ new TestingCharTokenizerNormalizeIsTokenChar(TEST_VERSION_CURRENT,
+ new StringReader(""));
+ fail("version 3.1 is not permitted if char based method is implemented");
+ } catch (IllegalArgumentException e) {
+ // expected
+ }
+ }
+
+ static final class TestingCharTokenizer extends CharTokenizer {
+ public TestingCharTokenizer(Version matchVersion, Reader input) {
+ super(matchVersion, input);
+ }
+
+ @Override
+ protected boolean isTokenChar(int c) {
+ return Character.isLetter(c);
+ }
+
+ @Deprecated @Override
+ protected boolean isTokenChar(char c) {
+ return Character.isLetter(c);
+ }
+ }
+
+ static final class TestingCharTokenizerNormalize extends CharTokenizer {
+ public TestingCharTokenizerNormalize(Version matchVersion, Reader input) {
+ super(matchVersion, input);
+ }
+
+ @Deprecated @Override
+ protected char normalize(char c) {
+ return c;
+ }
+
+ @Override
+ protected int normalize(int c) {
+ return c;
+ }
+ }
+
+ static final class TestingCharTokenizerNormalizeIsTokenChar extends CharTokenizer {
+ public TestingCharTokenizerNormalizeIsTokenChar(Version matchVersion,
+ Reader input) {
+ super(matchVersion, input);
+ }
+
+ @Deprecated @Override
+ protected char normalize(char c) {
+ return c;
+ }
+
+ @Override
+ protected int normalize(int c) {
+ return c;
+ }
+
+ @Override
+ protected boolean isTokenChar(int c) {
+ return Character.isLetter(c);
+ }
+
+ @Deprecated @Override
+ protected boolean isTokenChar(char c) {
+ return Character.isLetter(c);
+ }
+ }
+}
fnp / pylucene.git / blobdiff