X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java?ds=inline
diff --git a/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java b/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java
new file mode 100644
index 0000000..819a547
--- /dev/null
+++ b/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java
@@ -0,0 +1,121 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.Version;
+
+/**
+ * A WhitespaceTokenizer is a tokenizer that divides text at whitespace.
+ * Adjacent sequences of non-Whitespace characters form tokens.
+ *
+ * You must specify the required {@link Version} compatibility when creating
+ * {@link WhitespaceTokenizer}:
+ *
+ * - As of 3.1, {@link CharTokenizer} uses an int based API to normalize and
+ * detect token characters. See {@link CharTokenizer#isTokenChar(int)} and
+ * {@link CharTokenizer#normalize(int)} for details.
+ *
+ */
+public final class WhitespaceTokenizer extends CharTokenizer {
+
+ /**
+ * Construct a new WhitespaceTokenizer. * @param matchVersion Lucene version
+ * to match See {@link above}
+ *
+ * @param in
+ * the input to split up into tokens
+ */
+ public WhitespaceTokenizer(Version matchVersion, Reader in) {
+ super(matchVersion, in);
+ }
+
+ /**
+ * Construct a new WhitespaceTokenizer using a given {@link AttributeSource}.
+ *
+ * @param matchVersion
+ * Lucene version to match See {@link above}
+ * @param source
+ * the attribute source to use for this {@link Tokenizer}
+ * @param in
+ * the input to split up into tokens
+ */
+ public WhitespaceTokenizer(Version matchVersion, AttributeSource source, Reader in) {
+ super(matchVersion, source, in);
+ }
+
+ /**
+ * Construct a new WhitespaceTokenizer using a given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ *
+ * @param
+ * matchVersion Lucene version to match See
+ * {@link above}
+ * @param factory
+ * the attribute factory to use for this {@link Tokenizer}
+ * @param in
+ * the input to split up into tokens
+ */
+ public WhitespaceTokenizer(Version matchVersion, AttributeFactory factory, Reader in) {
+ super(matchVersion, factory, in);
+ }
+
+ /**
+ * Construct a new WhitespaceTokenizer.
+ *
+ * @deprecated use {@link #WhitespaceTokenizer(Version, Reader)} instead. This will
+ * be removed in Lucene 4.0.
+ */
+ @Deprecated
+ public WhitespaceTokenizer(Reader in) {
+ super(in);
+ }
+
+ /**
+ * Construct a new WhitespaceTokenizer using a given {@link AttributeSource}.
+ *
+ * @deprecated use {@link #WhitespaceTokenizer(Version, AttributeSource, Reader)}
+ * instead. This will be removed in Lucene 4.0.
+ */
+ @Deprecated
+ public WhitespaceTokenizer(AttributeSource source, Reader in) {
+ super(source, in);
+ }
+
+ /**
+ * Construct a new WhitespaceTokenizer using a given
+ * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}.
+ *
+ * @deprecated use {@link #WhitespaceTokenizer(Version, AttributeSource.AttributeFactory, Reader)}
+ * instead. This will be removed in Lucene 4.0.
+ */
+ @Deprecated
+ public WhitespaceTokenizer(AttributeFactory factory, Reader in) {
+ super(factory, in);
+ }
+
+ /** Collects only characters which do not satisfy
+ * {@link Character#isWhitespace(int)}.*/
+ @Override
+ protected boolean isTokenChar(int c) {
+ return !Character.isWhitespace(c);
+ }
+}