pylucene 3.5.0-3

[pylucene.git] / lucene-java-3.5.0 / lucene / contrib / analyzers / common / src / java / org / apache / lucene / analysis / path / PathHierarchyTokenizer.java

diff --git a/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java b/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java
new file mode 100644 (file)
index 0000000..608c386
--- /dev/null
+++ b/lucene-java-3.5.0/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizer.java
@@ -0,0 +1,191 @@
+package org.apache.lucene.analysis.path;
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+
+/**
+ *
+ * Take something like:
+ *
+ * <pre>
+ *  /something/something/else
+ * </pre>
+ *
+ * and make:
+ *
+ * <pre>
+ *  /something
+ *  /something/something
+ *  /something/something/else
+ * </pre>
+ */
+public class PathHierarchyTokenizer extends Tokenizer {
+
+  public PathHierarchyTokenizer(Reader input) {
+    this(input, DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, DEFAULT_SKIP);
+  }
+
+  public PathHierarchyTokenizer(Reader input, int skip) {
+    this(input, DEFAULT_BUFFER_SIZE, DEFAULT_DELIMITER, DEFAULT_DELIMITER, skip);
+  }
+
+  public PathHierarchyTokenizer(Reader input, int bufferSize, char delimiter) {
+    this(input, bufferSize, delimiter, delimiter, DEFAULT_SKIP);
+  }
+
+  public PathHierarchyTokenizer(Reader input, char delimiter, char replacement) {
+    this(input, DEFAULT_BUFFER_SIZE, delimiter, replacement, DEFAULT_SKIP);
+  }
+
+  public PathHierarchyTokenizer(Reader input, char delimiter, char replacement, int skip) {
+    this(input, DEFAULT_BUFFER_SIZE, delimiter, replacement, skip);
+  }
+
+  public PathHierarchyTokenizer(Reader input, int bufferSize, char delimiter, char replacement, int skip) {
+    super(input);
+    termAtt.resizeBuffer(bufferSize);
+
+    this.delimiter = delimiter;
+    this.replacement = replacement;
+    this.skip = skip;
+    resultToken = new StringBuilder(bufferSize);
+  }
+
+  private static final int DEFAULT_BUFFER_SIZE = 1024;
+  public static final char DEFAULT_DELIMITER = '/';
+  public static final int DEFAULT_SKIP = 0;
+
+  private final char delimiter;
+  private final char replacement;
+  private final int skip;
+
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+  private final PositionIncrementAttribute posAtt = addAttribute(PositionIncrementAttribute.class);
+  private int startPosition = 0;
+  private int finalOffset = 0;
+  private int skipped = 0;
+  private boolean endDelimiter = false;
+  private StringBuilder resultToken;
+
+
+  @Override
+  public final boolean incrementToken() throws IOException {
+    clearAttributes();
+    termAtt.append( resultToken );
+    if(resultToken.length() == 0){
+      posAtt.setPositionIncrement(1);
+    }
+    else{
+      posAtt.setPositionIncrement(0);
+    }
+    int length = 0;
+    boolean added = false;
+    if( endDelimiter ){
+      termAtt.append(replacement);
+      length++;
+      endDelimiter = false;
+      added = true;
+    }
+
+    while (true) {
+      int c = input.read();
+      if( c < 0 ){
+        if( skipped > skip ) {
+          length += resultToken.length();
+          termAtt.setLength(length);
+          finalOffset = correctOffset(startPosition + length);
+          offsetAtt.setOffset(correctOffset(startPosition), finalOffset);
+          if( added ){
+            resultToken.setLength(0);
+            resultToken.append(termAtt.buffer(), 0, length);
+          }
+          return added;
+        }
+        else{
+          finalOffset = correctOffset(startPosition + length);
+          return false;
+        }
+      }
+      if( !added ){
+        added = true;
+        skipped++;
+        if( skipped > skip ){
+          termAtt.append(c == delimiter ? replacement : (char)c);
+          length++;
+        }
+        else {
+          startPosition++;
+        }
+      }
+      else {
+        if( c == delimiter ){
+          if( skipped > skip ){
+            endDelimiter = true;
+            break;
+          }
+          skipped++;
+          if( skipped > skip ){
+            termAtt.append(replacement);
+            length++;
+          }
+          else {
+            startPosition++;
+          }
+        }
+        else {
+          if( skipped > skip ){
+            termAtt.append((char)c);
+            length++;
+          }
+          else {
+            startPosition++;
+          }
+        }
+      }
+    }
+    length += resultToken.length();
+    termAtt.setLength(length);
+    finalOffset = correctOffset(startPosition + length);
+    offsetAtt.setOffset(correctOffset(startPosition), finalOffset);
+    resultToken.setLength(0);
+    resultToken.append(termAtt.buffer(), 0, length);
+    return true;
+  }
+
+  @Override
+  public final void end() {
+    // set final offset
+    offsetAtt.setOffset(finalOffset, finalOffset);
+  }
+
+  @Override
+  public void reset(Reader input) throws IOException {
+    super.reset(input);
+    resultToken.setLength(0);
+    finalOffset = 0;
+    endDelimiter = false;
+    skipped = 0;
+  }
+}