--- /dev/null
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.util.AttributeSource.AttributeFactory;
+
+/**
+ * Tokenizer for testing.
+ * <p>
+ * This tokenizer is a replacement for {@link #WHITESPACE}, {@link #SIMPLE}, and {@link #KEYWORD}
+ * tokenizers. If you are writing a component such as a TokenFilter, its a great idea to test
+ * it wrapping this tokenizer instead for extra checks. This tokenizer has the following behavior:
+ * <ul>
+ * <li>An internal state-machine is used for checking consumer consistency. These checks can
+ * be disabled with {@link #setEnableChecks(boolean)}.
+ * <li>For convenience, optionally lowercases terms that it outputs.
+ * </ul>
+ */
+public class MockTokenizer extends Tokenizer {
+ /** Acts Similar to WhitespaceTokenizer */
+ public static final int WHITESPACE = 0;
+ /** Acts Similar to KeywordTokenizer.
+ * TODO: Keyword returns an "empty" token for an empty reader...
+ */
+ public static final int KEYWORD = 1;
+ /** Acts like LetterTokenizer. */
+ public static final int SIMPLE = 2;
+
+ private final int pattern;
+ private final boolean lowerCase;
+ private final int maxTokenLength;
+ public static final int DEFAULT_MAX_TOKEN_LENGTH = Integer.MAX_VALUE;
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ int off = 0;
+
+ // TODO: "register" with LuceneTestCase to ensure all streams are closed() ?
+ // currently, we can only check that the lifecycle is correct if someone is reusing,
+ // but not for "one-offs".
+ private static enum State {
+ SETREADER, // consumer set a reader input either via ctor or via reset(Reader)
+ RESET, // consumer has called reset()
+ INCREMENT, // consumer is consuming, has called incrementToken() == true
+ INCREMENT_FALSE, // consumer has called incrementToken() which returned false
+ END, // consumer has called end() to perform end of stream operations
+ CLOSE // consumer has called close() to release any resources
+ };
+
+ private State streamState = State.CLOSE;
+ private boolean enableChecks = true;
+
+ public MockTokenizer(AttributeFactory factory, Reader input, int pattern, boolean lowerCase, int maxTokenLength) {
+ super(factory, input);
+ this.pattern = pattern;
+ this.lowerCase = lowerCase;
+ this.streamState = State.SETREADER;
+ this.maxTokenLength = maxTokenLength;
+ }
+
+ public MockTokenizer(Reader input, int pattern, boolean lowerCase, int maxTokenLength) {
+ this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, pattern, lowerCase, maxTokenLength);
+ }
+
+ public MockTokenizer(Reader input, int pattern, boolean lowerCase) {
+ this(input, pattern, lowerCase, DEFAULT_MAX_TOKEN_LENGTH);
+ }
+
+ @Override
+ public final boolean incrementToken() throws IOException {
+ assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT)
+ : "incrementToken() called while in wrong state: " + streamState;
+ clearAttributes();
+ for (;;) {
+ int startOffset = off;
+ int cp = readCodePoint();
+ if (cp < 0) {
+ break;
+ } else if (isTokenChar(cp)) {
+ int endOffset;
+ do {
+ char chars[] = Character.toChars(normalize(cp));
+ for (int i = 0; i < chars.length; i++)
+ termAtt.append(chars[i]);
+ endOffset = off;
+ if (termAtt.length() >= maxTokenLength) {
+ break;
+ }
+ cp = readCodePoint();
+ } while (cp >= 0 && isTokenChar(cp));
+ offsetAtt.setOffset(correctOffset(startOffset), correctOffset(endOffset));
+ streamState = State.INCREMENT;
+ return true;
+ }
+ }
+ streamState = State.INCREMENT_FALSE;
+ return false;
+ }
+
+ protected int readCodePoint() throws IOException {
+ int ch = input.read();
+ if (ch < 0) {
+ return ch;
+ } else {
+ assert ch != 0xffff; /* only on 3.x */
+ assert !Character.isLowSurrogate((char) ch);
+ off++;
+ if (Character.isHighSurrogate((char) ch)) {
+ int ch2 = input.read();
+ if (ch2 >= 0) {
+ off++;
+ assert Character.isLowSurrogate((char) ch2);
+ return Character.toCodePoint((char) ch, (char) ch2);
+ }
+ }
+ return ch;
+ }
+ }
+
+ protected boolean isTokenChar(int c) {
+ switch(pattern) {
+ case WHITESPACE: return !Character.isWhitespace(c);
+ case KEYWORD: return true;
+ case SIMPLE: return Character.isLetter(c);
+ default: throw new RuntimeException("invalid pattern constant:" + pattern);
+ }
+ }
+
+ protected int normalize(int c) {
+ return lowerCase ? Character.toLowerCase(c) : c;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ off = 0;
+ assert !enableChecks || streamState != State.RESET : "double reset()";
+ streamState = State.RESET;
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ // in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely close()
+ // these tests should disable this check, by default we check the normal workflow.
+ // TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this
+ assert !enableChecks || streamState == State.END || streamState == State.CLOSE : "close() called in wrong state: " + streamState;
+ streamState = State.CLOSE;
+ }
+
+ @Override
+ public void reset(Reader input) throws IOException {
+ super.reset(input);
+ assert !enableChecks || streamState == State.CLOSE : "setReader() called in wrong state: " + streamState;
+ streamState = State.SETREADER;
+ }
+
+ @Override
+ public void end() throws IOException {
+ int finalOffset = correctOffset(off);
+ offsetAtt.setOffset(finalOffset, finalOffset);
+ // some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false.
+ // these tests should disable this check (in general you should consume the entire stream)
+ assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!";
+ streamState = State.END;
+ }
+
+ /**
+ * Toggle consumer workflow checking: if your test consumes tokenstreams normally you
+ * should leave this enabled.
+ */
+ public void setEnableChecks(boolean enableChecks) {
+ this.enableChecks = enableChecks;
+ }
+}
fnp / pylucene.git / blobdiff