+++ /dev/null
-package org.apache.lucene.analysis;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-import java.io.StringReader;
-import java.io.Reader;
-
-import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import org.apache.lucene.index.Payload;
-
-public class TestAnalyzers extends BaseTokenStreamTestCase {
-
- public void testSimple() throws Exception {
- Analyzer a = new SimpleAnalyzer(TEST_VERSION_CURRENT);
- assertAnalyzesTo(a, "foo bar FOO BAR",
- new String[] { "foo", "bar", "foo", "bar" });
- assertAnalyzesTo(a, "foo bar . FOO <> BAR",
- new String[] { "foo", "bar", "foo", "bar" });
- assertAnalyzesTo(a, "foo.bar.FOO.BAR",
- new String[] { "foo", "bar", "foo", "bar" });
- assertAnalyzesTo(a, "U.S.A.",
- new String[] { "u", "s", "a" });
- assertAnalyzesTo(a, "C++",
- new String[] { "c" });
- assertAnalyzesTo(a, "B2B",
- new String[] { "b", "b" });
- assertAnalyzesTo(a, "2B",
- new String[] { "b" });
- assertAnalyzesTo(a, "\"QUOTED\" word",
- new String[] { "quoted", "word" });
- }
-
- public void testNull() throws Exception {
- Analyzer a = new WhitespaceAnalyzer(TEST_VERSION_CURRENT);
- assertAnalyzesTo(a, "foo bar FOO BAR",
- new String[] { "foo", "bar", "FOO", "BAR" });
- assertAnalyzesTo(a, "foo bar . FOO <> BAR",
- new String[] { "foo", "bar", ".", "FOO", "<>", "BAR" });
- assertAnalyzesTo(a, "foo.bar.FOO.BAR",
- new String[] { "foo.bar.FOO.BAR" });
- assertAnalyzesTo(a, "U.S.A.",
- new String[] { "U.S.A." });
- assertAnalyzesTo(a, "C++",
- new String[] { "C++" });
- assertAnalyzesTo(a, "B2B",
- new String[] { "B2B" });
- assertAnalyzesTo(a, "2B",
- new String[] { "2B" });
- assertAnalyzesTo(a, "\"QUOTED\" word",
- new String[] { "\"QUOTED\"", "word" });
- }
-
- public void testStop() throws Exception {
- Analyzer a = new StopAnalyzer(TEST_VERSION_CURRENT);
- assertAnalyzesTo(a, "foo bar FOO BAR",
- new String[] { "foo", "bar", "foo", "bar" });
- assertAnalyzesTo(a, "foo a bar such FOO THESE BAR",
- new String[] { "foo", "bar", "foo", "bar" });
- }
-
- void verifyPayload(TokenStream ts) throws IOException {
- PayloadAttribute payloadAtt = ts.getAttribute(PayloadAttribute.class);
- for(byte b=1;;b++) {
- boolean hasNext = ts.incrementToken();
- if (!hasNext) break;
- // System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
- // System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
- assertEquals(b, payloadAtt.getPayload().toByteArray()[0]);
- }
- }
-
- // Make sure old style next() calls result in a new copy of payloads
- public void testPayloadCopy() throws IOException {
- String s = "how now brown cow";
- TokenStream ts;
- ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(s));
- ts = new PayloadSetter(ts);
- verifyPayload(ts);
-
- ts = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader(s));
- ts = new PayloadSetter(ts);
- verifyPayload(ts);
- }
-
- // LUCENE-1150: Just a compile time test, to ensure the
- // StandardAnalyzer constants remain publicly accessible
- @SuppressWarnings("unused")
- public void _testStandardConstants() {
- int x = StandardTokenizer.ALPHANUM;
- x = StandardTokenizer.APOSTROPHE;
- x = StandardTokenizer.ACRONYM;
- x = StandardTokenizer.COMPANY;
- x = StandardTokenizer.EMAIL;
- x = StandardTokenizer.HOST;
- x = StandardTokenizer.NUM;
- x = StandardTokenizer.CJ;
- String[] y = StandardTokenizer.TOKEN_TYPES;
- }
-
- private static class LowerCaseWhitespaceAnalyzer extends Analyzer {
-
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new LowerCaseFilter(TEST_VERSION_CURRENT,
- new WhitespaceTokenizer(TEST_VERSION_CURRENT, reader));
- }
-
- }
-
- /**
- * @deprecated remove this when lucene 3.0 "broken unicode 4" support
- * is no longer needed.
- */
- @Deprecated
- private static class LowerCaseWhitespaceAnalyzerBWComp extends Analyzer {
-
- @Override
- public TokenStream tokenStream(String fieldName, Reader reader) {
- return new LowerCaseFilter(new WhitespaceTokenizer(reader));
- }
-
- }
-
- /**
- * Test that LowercaseFilter handles entire unicode range correctly
- */
- public void testLowerCaseFilter() throws IOException {
- Analyzer a = new LowerCaseWhitespaceAnalyzer();
- // BMP
- assertAnalyzesTo(a, "AbaCaDabA", new String[] { "abacadaba" });
- // supplementary
- assertAnalyzesTo(a, "\ud801\udc16\ud801\udc16\ud801\udc16\ud801\udc16",
- new String[] {"\ud801\udc3e\ud801\udc3e\ud801\udc3e\ud801\udc3e"});
- assertAnalyzesTo(a, "AbaCa\ud801\udc16DabA",
- new String[] { "abaca\ud801\udc3edaba" });
- // unpaired lead surrogate
- assertAnalyzesTo(a, "AbaC\uD801AdaBa",
- new String [] { "abac\uD801adaba" });
- // unpaired trail surrogate
- assertAnalyzesTo(a, "AbaC\uDC16AdaBa",
- new String [] { "abac\uDC16adaba" });
- }
-
- /**
- * Test that LowercaseFilter handles the lowercasing correctly if the term
- * buffer has a trailing surrogate character leftover and the current term in
- * the buffer ends with a corresponding leading surrogate.
- */
- public void testLowerCaseFilterLowSurrogateLeftover() throws IOException {
- // test if the limit of the termbuffer is correctly used with supplementary
- // chars
- WhitespaceTokenizer tokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT,
- new StringReader("BogustermBogusterm\udc16"));
- LowerCaseFilter filter = new LowerCaseFilter(TEST_VERSION_CURRENT,
- tokenizer);
- assertTokenStreamContents(filter, new String[] {"bogustermbogusterm\udc16"});
- filter.reset();
- String highSurEndingUpper = "BogustermBoguster\ud801";
- String highSurEndingLower = "bogustermboguster\ud801";
- tokenizer.reset(new StringReader(highSurEndingUpper));
- assertTokenStreamContents(filter, new String[] {highSurEndingLower});
- assertTrue(filter.hasAttribute(CharTermAttribute.class));
- char[] termBuffer = filter.getAttribute(CharTermAttribute.class).buffer();
- int length = highSurEndingLower.length();
- assertEquals('\ud801', termBuffer[length - 1]);
- assertEquals('\udc3e', termBuffer[length]);
-
- }
-
- public void testLimitTokenCountAnalyzer() throws IOException {
- Analyzer a = new LimitTokenCountAnalyzer(new WhitespaceAnalyzer(TEST_VERSION_CURRENT), 2);
- // dont use assertAnalyzesTo here, as the end offset is not the end of the string!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 3 }, new int[] { 1, 4 }, 4);
- assertTokenStreamContents(a.reusableTokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
-
- a = new LimitTokenCountAnalyzer(new StandardAnalyzer(TEST_VERSION_CURRENT), 2);
- // dont use assertAnalyzesTo here, as the end offset is not the end of the string!
- assertTokenStreamContents(a.tokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
- assertTokenStreamContents(a.reusableTokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
- }
-
- /**
- * Test that LowercaseFilter only works on BMP for back compat,
- * depending upon version
- * @deprecated remove this test when lucene 3.0 "broken unicode 4" support
- * is no longer needed.
- */
- @Deprecated
- public void testLowerCaseFilterBWComp() throws IOException {
- Analyzer a = new LowerCaseWhitespaceAnalyzerBWComp();
- // BMP
- assertAnalyzesTo(a, "AbaCaDabA", new String[] { "abacadaba" });
- // supplementary, no-op
- assertAnalyzesTo(a, "\ud801\udc16\ud801\udc16\ud801\udc16\ud801\udc16",
- new String[] {"\ud801\udc16\ud801\udc16\ud801\udc16\ud801\udc16"});
- assertAnalyzesTo(a, "AbaCa\ud801\udc16DabA",
- new String[] { "abaca\ud801\udc16daba" });
- // unpaired lead surrogate
- assertAnalyzesTo(a, "AbaC\uD801AdaBa",
- new String [] { "abac\uD801adaba" });
- // unpaired trail surrogate
- assertAnalyzesTo(a, "AbaC\uDC16AdaBa",
- new String [] { "abac\uDC16adaba" });
- }
-
- /** blast some random strings through the analyzer */
- public void testRandomStrings() throws Exception {
- checkRandomData(random, new WhitespaceAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
- checkRandomData(random, new SimpleAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
- checkRandomData(random, new StopAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
- }
-}
-
-final class PayloadSetter extends TokenFilter {
- PayloadAttribute payloadAtt;
- public PayloadSetter(TokenStream input) {
- super(input);
- payloadAtt = addAttribute(PayloadAttribute.class);
- }
-
- byte[] data = new byte[1];
- Payload p = new Payload(data,0,1);
-
- @Override
- public boolean incrementToken() throws IOException {
- boolean hasNext = input.incrementToken();
- if (!hasNext) return false;
- payloadAtt.setPayload(p); // reuse the payload / byte[]
- data[0]++;
- return true;
- }
-}
\ No newline at end of file
fnp / pylucene.git / blobdiff