+++ /dev/null
-package org.apache.lucene.analysis.nl;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.util.Map;
-
-/**
- * A stemmer for Dutch words.
- * <p>
- * The algorithm is an implementation of
- * the <a href="http://snowball.tartarus.org/algorithms/dutch/stemmer.html">dutch stemming</a>
- * algorithm in Martin Porter's snowball project.
- * </p>
- * @deprecated Use {@link org.tartarus.snowball.ext.DutchStemmer} instead,
- * which has the same functionality. This filter will be removed in Lucene 5.0
- */
-@Deprecated
-public class DutchStemmer {
- /**
- * Buffer for the terms while stemming them.
- */
- private StringBuilder sb = new StringBuilder();
- private boolean _removedE;
- private Map _stemDict;
-
- private int _R1;
- private int _R2;
-
- //TODO convert to internal
- /*
- * Stems the given term to an unique <tt>discriminator</tt>.
- *
- * @param term The term that should be stemmed.
- * @return Discriminator for <tt>term</tt>
- */
- public String stem(String term) {
- term = term.toLowerCase();
- if (!isStemmable(term))
- return term;
- if (_stemDict != null && _stemDict.containsKey(term))
- if (_stemDict.get(term) instanceof String)
- return (String) _stemDict.get(term);
- else
- return null;
-
- // Reset the StringBuilder.
- sb.delete(0, sb.length());
- sb.insert(0, term);
- // Stemming starts here...
- substitute(sb);
- storeYandI(sb);
- _R1 = getRIndex(sb, 0);
- _R1 = Math.max(3, _R1);
- step1(sb);
- step2(sb);
- _R2 = getRIndex(sb, _R1);
- step3a(sb);
- step3b(sb);
- step4(sb);
- reStoreYandI(sb);
- return sb.toString();
- }
-
- private boolean enEnding(StringBuilder sb) {
- String[] enend = new String[]{"ene", "en"};
- for (int i = 0; i < enend.length; i++) {
- String end = enend[i];
- String s = sb.toString();
- int index = s.length() - end.length();
- if (s.endsWith(end) &&
- index >= _R1 &&
- isValidEnEnding(sb, index - 1)
- ) {
- sb.delete(index, index + end.length());
- unDouble(sb, index);
- return true;
- }
- }
- return false;
- }
-
-
- private void step1(StringBuilder sb) {
- if (_R1 >= sb.length())
- return;
-
- String s = sb.toString();
- int lengthR1 = sb.length() - _R1;
- int index;
-
- if (s.endsWith("heden")) {
- sb.replace(_R1, lengthR1 + _R1, sb.substring(_R1, lengthR1 + _R1).replaceAll("heden", "heid"));
- return;
- }
-
- if (enEnding(sb))
- return;
-
- if (s.endsWith("se") &&
- (index = s.length() - 2) >= _R1 &&
- isValidSEnding(sb, index - 1)
- ) {
- sb.delete(index, index + 2);
- return;
- }
- if (s.endsWith("s") &&
- (index = s.length() - 1) >= _R1 &&
- isValidSEnding(sb, index - 1)) {
- sb.delete(index, index + 1);
- }
- }
-
- /**
- * Delete suffix e if in R1 and
- * preceded by a non-vowel, and then undouble the ending
- *
- * @param sb String being stemmed
- */
- private void step2(StringBuilder sb) {
- _removedE = false;
- if (_R1 >= sb.length())
- return;
- String s = sb.toString();
- int index = s.length() - 1;
- if (index >= _R1 &&
- s.endsWith("e") &&
- !isVowel(sb.charAt(index - 1))) {
- sb.delete(index, index + 1);
- unDouble(sb);
- _removedE = true;
- }
- }
-
- /**
- * Delete "heid"
- *
- * @param sb String being stemmed
- */
- private void step3a(StringBuilder sb) {
- if (_R2 >= sb.length())
- return;
- String s = sb.toString();
- int index = s.length() - 4;
- if (s.endsWith("heid") && index >= _R2 && sb.charAt(index - 1) != 'c') {
- sb.delete(index, index + 4); //remove heid
- enEnding(sb);
- }
- }
-
- /**
- * <p>A d-suffix, or derivational suffix, enables a new word,
- * often with a different grammatical category, or with a different
- * sense, to be built from another word. Whether a d-suffix can be
- * attached is discovered not from the rules of grammar, but by
- * referring to a dictionary. So in English, ness can be added to
- * certain adjectives to form corresponding nouns (littleness,
- * kindness, foolishness ...) but not to all adjectives
- * (not for example, to big, cruel, wise ...) d-suffixes can be
- * used to change meaning, often in rather exotic ways.</p>
- * Remove "ing", "end", "ig", "lijk", "baar" and "bar"
- *
- * @param sb String being stemmed
- */
- private void step3b(StringBuilder sb) {
- if (_R2 >= sb.length())
- return;
- String s = sb.toString();
- int index = 0;
-
- if ((s.endsWith("end") || s.endsWith("ing")) &&
- (index = s.length() - 3) >= _R2) {
- sb.delete(index, index + 3);
- if (sb.charAt(index - 2) == 'i' &&
- sb.charAt(index - 1) == 'g') {
- if (sb.charAt(index - 3) != 'e' & index - 2 >= _R2) {
- index -= 2;
- sb.delete(index, index + 2);
- }
- } else {
- unDouble(sb, index);
- }
- return;
- }
- if (s.endsWith("ig") &&
- (index = s.length() - 2) >= _R2
- ) {
- if (sb.charAt(index - 1) != 'e')
- sb.delete(index, index + 2);
- return;
- }
- if (s.endsWith("lijk") &&
- (index = s.length() - 4) >= _R2
- ) {
- sb.delete(index, index + 4);
- step2(sb);
- return;
- }
- if (s.endsWith("baar") &&
- (index = s.length() - 4) >= _R2
- ) {
- sb.delete(index, index + 4);
- return;
- }
- if (s.endsWith("bar") &&
- (index = s.length() - 3) >= _R2
- ) {
- if (_removedE)
- sb.delete(index, index + 3);
- return;
- }
- }
-
- /**
- * undouble vowel
- * If the words ends CVD, where C is a non-vowel, D is a non-vowel other than I, and V is double a, e, o or u, remove one of the vowels from V (for example, maan -> man, brood -> brod).
- *
- * @param sb String being stemmed
- */
- private void step4(StringBuilder sb) {
- if (sb.length() < 4)
- return;
- String end = sb.substring(sb.length() - 4, sb.length());
- char c = end.charAt(0);
- char v1 = end.charAt(1);
- char v2 = end.charAt(2);
- char d = end.charAt(3);
- if (v1 == v2 &&
- d != 'I' &&
- v1 != 'i' &&
- isVowel(v1) &&
- !isVowel(d) &&
- !isVowel(c)) {
- sb.delete(sb.length() - 2, sb.length() - 1);
- }
- }
-
- /**
- * Checks if a term could be stemmed.
- *
- * @return true if, and only if, the given term consists in letters.
- */
- private boolean isStemmable(String term) {
- for (int c = 0; c < term.length(); c++) {
- if (!Character.isLetter(term.charAt(c))) return false;
- }
- return true;
- }
-
- /**
- * Substitute ä, ë, ï, ö, ü, á , é, í, ó, ú
- */
- private void substitute(StringBuilder buffer) {
- for (int i = 0; i < buffer.length(); i++) {
- switch (buffer.charAt(i)) {
- case 'ä':
- case 'á':
- {
- buffer.setCharAt(i, 'a');
- break;
- }
- case 'ë':
- case 'é':
- {
- buffer.setCharAt(i, 'e');
- break;
- }
- case 'ü':
- case 'ú':
- {
- buffer.setCharAt(i, 'u');
- break;
- }
- case 'ï':
- case 'i':
- {
- buffer.setCharAt(i, 'i');
- break;
- }
- case 'ö':
- case 'ó':
- {
- buffer.setCharAt(i, 'o');
- break;
- }
- }
- }
- }
-
- /*private boolean isValidSEnding(StringBuilder sb) {
- return isValidSEnding(sb, sb.length() - 1);
- }*/
-
- private boolean isValidSEnding(StringBuilder sb, int index) {
- char c = sb.charAt(index);
- if (isVowel(c) || c == 'j')
- return false;
- return true;
- }
-
- /*private boolean isValidEnEnding(StringBuilder sb) {
- return isValidEnEnding(sb, sb.length() - 1);
- }*/
-
- private boolean isValidEnEnding(StringBuilder sb, int index) {
- char c = sb.charAt(index);
- if (isVowel(c))
- return false;
- if (c < 3)
- return false;
- // ends with "gem"?
- if (c == 'm' && sb.charAt(index - 2) == 'g' && sb.charAt(index - 1) == 'e')
- return false;
- return true;
- }
-
- private void unDouble(StringBuilder sb) {
- unDouble(sb, sb.length());
- }
-
- private void unDouble(StringBuilder sb, int endIndex) {
- String s = sb.substring(0, endIndex);
- if (s.endsWith("kk") || s.endsWith("tt") || s.endsWith("dd") || s.endsWith("nn") || s.endsWith("mm") || s.endsWith("ff")) {
- sb.delete(endIndex - 1, endIndex);
- }
- }
-
- private int getRIndex(StringBuilder sb, int start) {
- if (start == 0)
- start = 1;
- int i = start;
- for (; i < sb.length(); i++) {
- //first non-vowel preceded by a vowel
- if (!isVowel(sb.charAt(i)) && isVowel(sb.charAt(i - 1))) {
- return i + 1;
- }
- }
- return i + 1;
- }
-
- private void storeYandI(StringBuilder sb) {
- if (sb.charAt(0) == 'y')
- sb.setCharAt(0, 'Y');
-
- int last = sb.length() - 1;
-
- for (int i = 1; i < last; i++) {
- switch (sb.charAt(i)) {
- case 'i':
- {
- if (isVowel(sb.charAt(i - 1)) &&
- isVowel(sb.charAt(i + 1))
- )
- sb.setCharAt(i, 'I');
- break;
- }
- case 'y':
- {
- if (isVowel(sb.charAt(i - 1)))
- sb.setCharAt(i, 'Y');
- break;
- }
- }
- }
- if (last > 0 && sb.charAt(last) == 'y' && isVowel(sb.charAt(last - 1)))
- sb.setCharAt(last, 'Y');
- }
-
- private void reStoreYandI(StringBuilder sb) {
- String tmp = sb.toString();
- sb.delete(0, sb.length());
- sb.insert(0, tmp.replaceAll("I", "i").replaceAll("Y", "y"));
- }
-
- private boolean isVowel(char c) {
- switch (c) {
- case 'e':
- case 'a':
- case 'o':
- case 'i':
- case 'u':
- case 'y':
- case 'è':
- {
- return true;
- }
- }
- return false;
- }
-
- void setStemDictionary(Map dict) {
- _stemDict = dict;
- }
-
-}
fnp / pylucene.git / blobdiff