--- /dev/null
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.index.Payload;
+import org.apache.lucene.analysis.tokenattributes.*;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util._TestUtil;
+
+import java.io.StringReader;
+import java.util.HashMap;
+
+public class TestToken extends LuceneTestCase {
+
+ public void testCtor() throws Exception {
+ Token t = new Token();
+ char[] content = "hello".toCharArray();
+ t.setTermBuffer(content, 0, content.length);
+ assertNotSame(t.termBuffer(), content);
+ assertEquals(0, t.startOffset());
+ assertEquals(0, t.endOffset());
+ assertEquals("hello", t.term());
+ assertEquals("word", t.type());
+ assertEquals(0, t.getFlags());
+
+ t = new Token(6, 22);
+ t.setTermBuffer(content, 0, content.length);
+ assertEquals("hello", t.term());
+ assertEquals("hello", t.toString());
+ assertEquals(6, t.startOffset());
+ assertEquals(22, t.endOffset());
+ assertEquals("word", t.type());
+ assertEquals(0, t.getFlags());
+
+ t = new Token(6, 22, 7);
+ t.setTermBuffer(content, 0, content.length);
+ assertEquals("hello", t.term());
+ assertEquals("hello", t.toString());
+ assertEquals(6, t.startOffset());
+ assertEquals(22, t.endOffset());
+ assertEquals("word", t.type());
+ assertEquals(7, t.getFlags());
+
+ t = new Token(6, 22, "junk");
+ t.setTermBuffer(content, 0, content.length);
+ assertEquals("hello", t.term());
+ assertEquals("hello", t.toString());
+ assertEquals(6, t.startOffset());
+ assertEquals(22, t.endOffset());
+ assertEquals("junk", t.type());
+ assertEquals(0, t.getFlags());
+ }
+
+ public void testResize() {
+ Token t = new Token();
+ char[] content = "hello".toCharArray();
+ t.setTermBuffer(content, 0, content.length);
+ for (int i = 0; i < 2000; i++)
+ {
+ t.resizeTermBuffer(i);
+ assertTrue(i <= t.termBuffer().length);
+ assertEquals("hello", t.term());
+ }
+ }
+
+ public void testGrow() {
+ Token t = new Token();
+ StringBuilder buf = new StringBuilder("ab");
+ for (int i = 0; i < 20; i++)
+ {
+ char[] content = buf.toString().toCharArray();
+ t.setTermBuffer(content, 0, content.length);
+ assertEquals(buf.length(), t.termLength());
+ assertEquals(buf.toString(), t.term());
+ buf.append(buf.toString());
+ }
+ assertEquals(1048576, t.termLength());
+
+ // now as a string, first variant
+ t = new Token();
+ buf = new StringBuilder("ab");
+ for (int i = 0; i < 20; i++)
+ {
+ String content = buf.toString();
+ t.setTermBuffer(content, 0, content.length());
+ assertEquals(content.length(), t.termLength());
+ assertEquals(content, t.term());
+ buf.append(content);
+ }
+ assertEquals(1048576, t.termLength());
+
+ // now as a string, second variant
+ t = new Token();
+ buf = new StringBuilder("ab");
+ for (int i = 0; i < 20; i++)
+ {
+ String content = buf.toString();
+ t.setTermBuffer(content);
+ assertEquals(content.length(), t.termLength());
+ assertEquals(content, t.term());
+ buf.append(content);
+ }
+ assertEquals(1048576, t.termLength());
+
+ // Test for slow growth to a long term
+ t = new Token();
+ buf = new StringBuilder("a");
+ for (int i = 0; i < 20000; i++)
+ {
+ String content = buf.toString();
+ t.setTermBuffer(content);
+ assertEquals(content.length(), t.termLength());
+ assertEquals(content, t.term());
+ buf.append("a");
+ }
+ assertEquals(20000, t.termLength());
+
+ // Test for slow growth to a long term
+ t = new Token();
+ buf = new StringBuilder("a");
+ for (int i = 0; i < 20000; i++)
+ {
+ String content = buf.toString();
+ t.setTermBuffer(content);
+ assertEquals(content.length(), t.termLength());
+ assertEquals(content, t.term());
+ buf.append("a");
+ }
+ assertEquals(20000, t.termLength());
+ }
+
+ public void testToString() throws Exception {
+ char[] b = {'a', 'l', 'o', 'h', 'a'};
+ Token t = new Token("", 0, 5);
+ t.setTermBuffer(b, 0, 5);
+ assertEquals("aloha", t.toString());
+
+ t.setTermBuffer("hi there");
+ assertEquals("hi there", t.toString());
+ }
+
+ public void testTermBufferEquals() throws Exception {
+ Token t1a = new Token();
+ char[] content1a = "hello".toCharArray();
+ t1a.setTermBuffer(content1a, 0, 5);
+ Token t1b = new Token();
+ char[] content1b = "hello".toCharArray();
+ t1b.setTermBuffer(content1b, 0, 5);
+ Token t2 = new Token();
+ char[] content2 = "hello2".toCharArray();
+ t2.setTermBuffer(content2, 0, 6);
+ assertTrue(t1a.equals(t1b));
+ assertFalse(t1a.equals(t2));
+ assertFalse(t2.equals(t1b));
+ }
+
+ public void testMixedStringArray() throws Exception {
+ Token t = new Token("hello", 0, 5);
+ assertEquals(t.termLength(), 5);
+ assertEquals(t.term(), "hello");
+ t.setTermBuffer("hello2");
+ assertEquals(t.termLength(), 6);
+ assertEquals(t.term(), "hello2");
+ t.setTermBuffer("hello3".toCharArray(), 0, 6);
+ assertEquals(t.term(), "hello3");
+
+ char[] buffer = t.termBuffer();
+ buffer[1] = 'o';
+ assertEquals(t.term(), "hollo3");
+ }
+
+ public void testClone() throws Exception {
+ Token t = new Token(0, 5);
+ char[] content = "hello".toCharArray();
+ t.setTermBuffer(content, 0, 5);
+ char[] buf = t.termBuffer();
+ Token copy = (Token) TestSimpleAttributeImpls.assertCloneIsEqual(t);
+ assertEquals(t.term(), copy.term());
+ assertNotSame(buf, copy.termBuffer());
+
+ Payload pl = new Payload(new byte[]{1,2,3,4});
+ t.setPayload(pl);
+ copy = (Token) TestSimpleAttributeImpls.assertCloneIsEqual(t);
+ assertEquals(pl, copy.getPayload());
+ assertNotSame(pl, copy.getPayload());
+ }
+
+ public void testCopyTo() throws Exception {
+ Token t = new Token();
+ Token copy = (Token) TestSimpleAttributeImpls.assertCopyIsEqual(t);
+ assertEquals("", t.term());
+ assertEquals("", copy.term());
+
+ t = new Token(0, 5);
+ char[] content = "hello".toCharArray();
+ t.setTermBuffer(content, 0, 5);
+ char[] buf = t.termBuffer();
+ copy = (Token) TestSimpleAttributeImpls.assertCopyIsEqual(t);
+ assertEquals(t.term(), copy.term());
+ assertNotSame(buf, copy.termBuffer());
+
+ Payload pl = new Payload(new byte[]{1,2,3,4});
+ t.setPayload(pl);
+ copy = (Token) TestSimpleAttributeImpls.assertCopyIsEqual(t);
+ assertEquals(pl, copy.getPayload());
+ assertNotSame(pl, copy.getPayload());
+ }
+
+ public interface SenselessAttribute extends Attribute {}
+
+ public static final class SenselessAttributeImpl extends AttributeImpl implements SenselessAttribute {
+ @Override
+ public void copyTo(AttributeImpl target) {}
+ @Override
+ public void clear() {}
+ @Override
+ public boolean equals(Object o) { return (o instanceof SenselessAttributeImpl); }
+ @Override
+ public int hashCode() { return 0; }
+ }
+
+ public void testTokenAttributeFactory() throws Exception {
+ TokenStream ts = new WhitespaceTokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, new StringReader("foo bar"));
+
+ assertTrue("SenselessAttribute is not implemented by SenselessAttributeImpl",
+ ts.addAttribute(SenselessAttribute.class) instanceof SenselessAttributeImpl);
+
+ assertTrue("CharTermAttribute is not implemented by Token",
+ ts.addAttribute(CharTermAttribute.class) instanceof Token);
+ assertTrue("OffsetAttribute is not implemented by Token",
+ ts.addAttribute(OffsetAttribute.class) instanceof Token);
+ assertTrue("FlagsAttribute is not implemented by Token",
+ ts.addAttribute(FlagsAttribute.class) instanceof Token);
+ assertTrue("PayloadAttribute is not implemented by Token",
+ ts.addAttribute(PayloadAttribute.class) instanceof Token);
+ assertTrue("PositionIncrementAttribute is not implemented by Token",
+ ts.addAttribute(PositionIncrementAttribute.class) instanceof Token);
+ assertTrue("TypeAttribute is not implemented by Token",
+ ts.addAttribute(TypeAttribute.class) instanceof Token);
+ }
+
+ public void testAttributeReflection() throws Exception {
+ Token t = new Token("foobar", 6, 22, 8);
+ _TestUtil.assertAttributeReflection(t,
+ new HashMap<String,Object>() {{
+ put(CharTermAttribute.class.getName() + "#term", "foobar");
+ put(OffsetAttribute.class.getName() + "#startOffset", 6);
+ put(OffsetAttribute.class.getName() + "#endOffset", 22);
+ put(PositionIncrementAttribute.class.getName() + "#positionIncrement", 1);
+ put(PayloadAttribute.class.getName() + "#payload", null);
+ put(TypeAttribute.class.getName() + "#type", TypeAttribute.DEFAULT_TYPE);
+ put(FlagsAttribute.class.getName() + "#flags", 8);
+ }});
+ }
+
+}
fnp / pylucene.git / blobdiff