--- /dev/null
+package org.apache.lucene.analysis;
+
+/**
+ * Copyright 2004 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.English;
+import java.io.IOException;
+import java.io.StringReader;
+
+
+/**
+ * tests for the TestTeeSinkTokenFilter
+ */
+public class TestTeeSinkTokenFilter extends BaseTokenStreamTestCase {
+ protected StringBuilder buffer1;
+ protected StringBuilder buffer2;
+ protected String[] tokens1;
+ protected String[] tokens2;
+
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ tokens1 = new String[]{"The", "quick", "Burgundy", "Fox", "jumped", "over", "the", "lazy", "Red", "Dogs"};
+ tokens2 = new String[]{"The", "Lazy", "Dogs", "should", "stay", "on", "the", "porch"};
+ buffer1 = new StringBuilder();
+
+ for (int i = 0; i < tokens1.length; i++) {
+ buffer1.append(tokens1[i]).append(' ');
+ }
+ buffer2 = new StringBuilder();
+ for (int i = 0; i < tokens2.length; i++) {
+ buffer2.append(tokens2[i]).append(' ');
+ }
+ }
+
+ static final TeeSinkTokenFilter.SinkFilter theFilter = new TeeSinkTokenFilter.SinkFilter() {
+ @Override
+ public boolean accept(AttributeSource a) {
+ CharTermAttribute termAtt = a.getAttribute(CharTermAttribute.class);
+ return termAtt.toString().equalsIgnoreCase("The");
+ }
+ };
+
+ static final TeeSinkTokenFilter.SinkFilter dogFilter = new TeeSinkTokenFilter.SinkFilter() {
+ @Override
+ public boolean accept(AttributeSource a) {
+ CharTermAttribute termAtt = a.getAttribute(CharTermAttribute.class);
+ return termAtt.toString().equalsIgnoreCase("Dogs");
+ }
+ };
+
+ public void testGeneral() throws IOException {
+ final TeeSinkTokenFilter source = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer1.toString()), MockTokenizer.WHITESPACE, false));
+ final TokenStream sink1 = source.newSinkTokenStream();
+ final TokenStream sink2 = source.newSinkTokenStream(theFilter);
+
+ source.addAttribute(CheckClearAttributesAttribute.class);
+ sink1.addAttribute(CheckClearAttributesAttribute.class);
+ sink2.addAttribute(CheckClearAttributesAttribute.class);
+
+ assertTokenStreamContents(source, tokens1);
+ assertTokenStreamContents(sink1, tokens1);
+ assertTokenStreamContents(sink2, new String[]{"The", "the"});
+ }
+
+ public void testMultipleSources() throws Exception {
+ final TeeSinkTokenFilter tee1 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer1.toString()), MockTokenizer.WHITESPACE, false));
+ final TeeSinkTokenFilter.SinkTokenStream dogDetector = tee1.newSinkTokenStream(dogFilter);
+ final TeeSinkTokenFilter.SinkTokenStream theDetector = tee1.newSinkTokenStream(theFilter);
+ tee1.reset();
+ final TokenStream source1 = new CachingTokenFilter(tee1);
+
+ tee1.addAttribute(CheckClearAttributesAttribute.class);
+ dogDetector.addAttribute(CheckClearAttributesAttribute.class);
+ theDetector.addAttribute(CheckClearAttributesAttribute.class);
+
+ final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(new MockTokenizer(new StringReader(buffer2.toString()), MockTokenizer.WHITESPACE, false));
+ tee2.addSinkTokenStream(dogDetector);
+ tee2.addSinkTokenStream(theDetector);
+ final TokenStream source2 = tee2;
+
+ assertTokenStreamContents(source1, tokens1);
+ assertTokenStreamContents(source2, tokens2);
+
+ assertTokenStreamContents(theDetector, new String[]{"The", "the", "The", "the"});
+ assertTokenStreamContents(dogDetector, new String[]{"Dogs", "Dogs"});
+
+ source1.reset();
+ TokenStream lowerCasing = new LowerCaseFilter(TEST_VERSION_CURRENT, source1);
+ String[] lowerCaseTokens = new String[tokens1.length];
+ for (int i = 0; i < tokens1.length; i++)
+ lowerCaseTokens[i] = tokens1[i].toLowerCase();
+ assertTokenStreamContents(lowerCasing, lowerCaseTokens);
+ }
+
+ /**
+ * Not an explicit test, just useful to print out some info on performance
+ *
+ * @throws Exception
+ */
+ public void performance() throws Exception {
+ int[] tokCount = {100, 500, 1000, 2000, 5000, 10000};
+ int[] modCounts = {1, 2, 5, 10, 20, 50, 100, 200, 500};
+ for (int k = 0; k < tokCount.length; k++) {
+ StringBuilder buffer = new StringBuilder();
+ System.out.println("-----Tokens: " + tokCount[k] + "-----");
+ for (int i = 0; i < tokCount[k]; i++) {
+ buffer.append(English.intToEnglish(i).toUpperCase()).append(' ');
+ }
+ //make sure we produce the same tokens
+ TeeSinkTokenFilter teeStream = new TeeSinkTokenFilter(new StandardFilter(new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))));
+ TokenStream sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(100));
+ teeStream.consumeAllTokens();
+ TokenStream stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))), 100);
+ CharTermAttribute tfTok = stream.addAttribute(CharTermAttribute.class);
+ CharTermAttribute sinkTok = sink.addAttribute(CharTermAttribute.class);
+ for (int i=0; stream.incrementToken(); i++) {
+ assertTrue(sink.incrementToken());
+ assertTrue(tfTok + " is not equal to " + sinkTok + " at token: " + i, tfTok.equals(sinkTok) == true);
+ }
+
+ //simulate two fields, each being analyzed once, for 20 documents
+ for (int j = 0; j < modCounts.length; j++) {
+ int tfPos = 0;
+ long start = System.currentTimeMillis();
+ for (int i = 0; i < 20; i++) {
+ stream = new StandardFilter(new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString())));
+ PositionIncrementAttribute posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class);
+ while (stream.incrementToken()) {
+ tfPos += posIncrAtt.getPositionIncrement();
+ }
+ stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))), modCounts[j]);
+ posIncrAtt = stream.getAttribute(PositionIncrementAttribute.class);
+ while (stream.incrementToken()) {
+ tfPos += posIncrAtt.getPositionIncrement();
+ }
+ }
+ long finish = System.currentTimeMillis();
+ System.out.println("ModCount: " + modCounts[j] + " Two fields took " + (finish - start) + " ms");
+ int sinkPos = 0;
+ //simulate one field with one sink
+ start = System.currentTimeMillis();
+ for (int i = 0; i < 20; i++) {
+ teeStream = new TeeSinkTokenFilter(new StandardFilter(new StandardTokenizer(TEST_VERSION_CURRENT, new StringReader(buffer.toString()))));
+ sink = teeStream.newSinkTokenStream(new ModuloSinkFilter(modCounts[j]));
+ PositionIncrementAttribute posIncrAtt = teeStream.getAttribute(PositionIncrementAttribute.class);
+ while (teeStream.incrementToken()) {
+ sinkPos += posIncrAtt.getPositionIncrement();
+ }
+ //System.out.println("Modulo--------");
+ posIncrAtt = sink.getAttribute(PositionIncrementAttribute.class);
+ while (sink.incrementToken()) {
+ sinkPos += posIncrAtt.getPositionIncrement();
+ }
+ }
+ finish = System.currentTimeMillis();
+ System.out.println("ModCount: " + modCounts[j] + " Tee fields took " + (finish - start) + " ms");
+ assertTrue(sinkPos + " does not equal: " + tfPos, sinkPos == tfPos);
+
+ }
+ System.out.println("- End Tokens: " + tokCount[k] + "-----");
+ }
+
+ }
+
+
+ class ModuloTokenFilter extends TokenFilter {
+
+ int modCount;
+
+ ModuloTokenFilter(TokenStream input, int mc) {
+ super(input);
+ modCount = mc;
+ }
+
+ int count = 0;
+
+ //return every 100 tokens
+ @Override
+ public boolean incrementToken() throws IOException {
+ boolean hasNext;
+ for (hasNext = input.incrementToken();
+ hasNext && count % modCount != 0;
+ hasNext = input.incrementToken()) {
+ count++;
+ }
+ count++;
+ return hasNext;
+ }
+ }
+
+ class ModuloSinkFilter extends TeeSinkTokenFilter.SinkFilter {
+ int count = 0;
+ int modCount;
+
+ ModuloSinkFilter(int mc) {
+ modCount = mc;
+ }
+
+ @Override
+ public boolean accept(AttributeSource a) {
+ boolean b = (a != null && count % modCount == 0);
+ count++;
+ return b;
+ }
+
+ }
+}
+
fnp / pylucene.git / blobdiff