X-Git-Url: https://git.mdrn.pl/pylucene.git/blobdiff_plain/a2e61f0c04805cfcb8706176758d1283c7e3a55c..aaeed5504b982cf3545252ab528713250aa33eed:/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/analysis/Analyzer.java?ds=sidebyside

diff --git a/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/analysis/Analyzer.java b/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/analysis/Analyzer.java
new file mode 100644
index 0000000..087df77
--- /dev/null
+++ b/lucene-java-3.5.0/lucene/src/java/org/apache/lucene/analysis/Analyzer.java
@@ -0,0 +1,155 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+import java.io.IOException;
+import java.io.Closeable;
+import java.lang.reflect.Modifier;
+
+import org.apache.lucene.util.CloseableThreadLocal;
+import org.apache.lucene.store.AlreadyClosedException;
+
+import org.apache.lucene.document.Fieldable;
+
+/** An Analyzer builds TokenStreams, which analyze text.  It thus represents a
+ *  policy for extracting index terms from text.
+ *  <p>
+ *  Typical implementations first build a Tokenizer, which breaks the stream of
+ *  characters from the Reader into raw Tokens.  One or more TokenFilters may
+ *  then be applied to the output of the Tokenizer.
+ * <p>The {@code Analyzer}-API in Lucene is based on the decorator pattern.
+ * Therefore all non-abstract subclasses must be final or their {@link #tokenStream}
+ * and {@link #reusableTokenStream} implementations must be final! This is checked
+ * when Java assertions are enabled.
+ */
+public abstract class Analyzer implements Closeable {
+
+  protected Analyzer() {
+    super();
+    assert assertFinal();
+  }
+  
+  private boolean assertFinal() {
+    try {
+      final Class<?> clazz = getClass();
+      if (!clazz.desiredAssertionStatus())
+        return true;
+      assert clazz.isAnonymousClass() ||
+        (clazz.getModifiers() & (Modifier.FINAL | Modifier.PRIVATE)) != 0 ||
+        (
+          Modifier.isFinal(clazz.getMethod("tokenStream", String.class, Reader.class).getModifiers()) &&
+          Modifier.isFinal(clazz.getMethod("reusableTokenStream", String.class, Reader.class).getModifiers())
+        ) :
+        "Analyzer implementation classes or at least their tokenStream() and reusableTokenStream() implementations must be final";
+      return true;
+    } catch (NoSuchMethodException nsme) {
+      return false;
+    }
+  }
+
+  /** Creates a TokenStream which tokenizes all the text in the provided
+   * Reader.  Must be able to handle null field name for
+   * backward compatibility.
+   */
+  public abstract TokenStream tokenStream(String fieldName, Reader reader);
+
+  /** Creates a TokenStream that is allowed to be re-used
+   *  from the previous time that the same thread called
+   *  this method.  Callers that do not need to use more
+   *  than one TokenStream at the same time from this
+   *  analyzer should use this method for better
+   *  performance.
+   */
+  public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException {
+    return tokenStream(fieldName, reader);
+  }
+
+  private CloseableThreadLocal<Object> tokenStreams = new CloseableThreadLocal<Object>();
+
+  /** Used by Analyzers that implement reusableTokenStream
+   *  to retrieve previously saved TokenStreams for re-use
+   *  by the same thread. */
+  protected Object getPreviousTokenStream() {
+    try {
+      return tokenStreams.get();
+    } catch (NullPointerException npe) {
+      if (tokenStreams == null) {
+        throw new AlreadyClosedException("this Analyzer is closed");
+      } else {
+        throw npe;
+      }
+    }
+  }
+
+  /** Used by Analyzers that implement reusableTokenStream
+   *  to save a TokenStream for later re-use by the same
+   *  thread. */
+  protected void setPreviousTokenStream(Object obj) {
+    try {
+      tokenStreams.set(obj);
+    } catch (NullPointerException npe) {
+      if (tokenStreams == null) {
+        throw new AlreadyClosedException("this Analyzer is closed");
+      } else {
+        throw npe;
+      }
+    }
+  }
+
+  /**
+   * Invoked before indexing a Fieldable instance if
+   * terms have already been added to that field.  This allows custom
+   * analyzers to place an automatic position increment gap between
+   * Fieldable instances using the same field name.  The default value
+   * position increment gap is 0.  With a 0 position increment gap and
+   * the typical default token position increment of 1, all terms in a field,
+   * including across Fieldable instances, are in successive positions, allowing
+   * exact PhraseQuery matches, for instance, across Fieldable instance boundaries.
+   *
+   * @param fieldName Fieldable name being indexed.
+   * @return position increment gap, added to the next token emitted from {@link #tokenStream(String,Reader)}
+   */
+  public int getPositionIncrementGap(String fieldName) {
+    return 0;
+  }
+
+  /**
+   * Just like {@link #getPositionIncrementGap}, except for
+   * Token offsets instead.  By default this returns 1 for
+   * tokenized fields and, as if the fields were joined
+   * with an extra space character, and 0 for un-tokenized
+   * fields.  This method is only called if the field
+   * produced at least one token for indexing.
+   *
+   * @param field the field just indexed
+   * @return offset gap, added to the next token emitted from {@link #tokenStream(String,Reader)}
+   */
+  public int getOffsetGap(Fieldable field) {
+    if (field.isTokenized())
+      return 1;
+    else
+      return 0;
+  }
+
+  /** Frees persistent resources used by this Analyzer */
+  public void close() {
+    tokenStreams.close();
+    tokenStreams = null;
+  }
+}