--- /dev/null
+package org.apache.lucene.analysis.de;
+// This file is encoded in UTF-8
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A stemmer for German words.
+ * <p>
+ * The algorithm is based on the report
+ * "A Fast and Simple Stemming Algorithm for German Words" by Jörg
+ * Caumanns (joerg.caumanns at isst.fhg.de).
+ * </p>
+ */
+public class GermanStemmer
+{
+ /**
+ * Buffer for the terms while stemming them.
+ */
+ private StringBuilder sb = new StringBuilder();
+
+ /**
+ * Amount of characters that are removed with <tt>substitute()</tt> while stemming.
+ */
+ private int substCount = 0;
+
+ /**
+ * Stemms the given term to an unique <tt>discriminator</tt>.
+ *
+ * @param term The term that should be stemmed.
+ * @return Discriminator for <tt>term</tt>
+ */
+ protected String stem( String term )
+ {
+ // Use lowercase for medium stemming.
+ term = term.toLowerCase();
+ if ( !isStemmable( term ) )
+ return term;
+ // Reset the StringBuilder.
+ sb.delete( 0, sb.length() );
+ sb.insert( 0, term );
+ // Stemming starts here...
+ substitute( sb );
+ strip( sb );
+ optimize( sb );
+ resubstitute( sb );
+ removeParticleDenotion( sb );
+ return sb.toString();
+ }
+
+ /**
+ * Checks if a term could be stemmed.
+ *
+ * @return true if, and only if, the given term consists in letters.
+ */
+ private boolean isStemmable( String term )
+ {
+ for ( int c = 0; c < term.length(); c++ ) {
+ if ( !Character.isLetter( term.charAt( c ) ) )
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * suffix stripping (stemming) on the current term. The stripping is reduced
+ * to the seven "base" suffixes "e", "s", "n", "t", "em", "er" and * "nd",
+ * from which all regular suffixes are build of. The simplification causes
+ * some overstemming, and way more irregular stems, but still provides unique.
+ * discriminators in the most of those cases.
+ * The algorithm is context free, except of the length restrictions.
+ */
+ private void strip( StringBuilder buffer )
+ {
+ boolean doMore = true;
+ while ( doMore && buffer.length() > 3 ) {
+ if ( ( buffer.length() + substCount > 5 ) &&
+ buffer.substring( buffer.length() - 2, buffer.length() ).equals( "nd" ) )
+ {
+ buffer.delete( buffer.length() - 2, buffer.length() );
+ }
+ else if ( ( buffer.length() + substCount > 4 ) &&
+ buffer.substring( buffer.length() - 2, buffer.length() ).equals( "em" ) ) {
+ buffer.delete( buffer.length() - 2, buffer.length() );
+ }
+ else if ( ( buffer.length() + substCount > 4 ) &&
+ buffer.substring( buffer.length() - 2, buffer.length() ).equals( "er" ) ) {
+ buffer.delete( buffer.length() - 2, buffer.length() );
+ }
+ else if ( buffer.charAt( buffer.length() - 1 ) == 'e' ) {
+ buffer.deleteCharAt( buffer.length() - 1 );
+ }
+ else if ( buffer.charAt( buffer.length() - 1 ) == 's' ) {
+ buffer.deleteCharAt( buffer.length() - 1 );
+ }
+ else if ( buffer.charAt( buffer.length() - 1 ) == 'n' ) {
+ buffer.deleteCharAt( buffer.length() - 1 );
+ }
+ // "t" occurs only as suffix of verbs.
+ else if ( buffer.charAt( buffer.length() - 1 ) == 't' ) {
+ buffer.deleteCharAt( buffer.length() - 1 );
+ }
+ else {
+ doMore = false;
+ }
+ }
+ }
+
+ /**
+ * Does some optimizations on the term. This optimisations are
+ * contextual.
+ */
+ private void optimize( StringBuilder buffer )
+ {
+ // Additional step for female plurals of professions and inhabitants.
+ if ( buffer.length() > 5 && buffer.substring( buffer.length() - 5, buffer.length() ).equals( "erin*" ) ) {
+ buffer.deleteCharAt( buffer.length() -1 );
+ strip( buffer );
+ }
+ // Additional step for irregular plural nouns like "Matrizen -> Matrix".
+ // NOTE: this length constraint is probably not a great value, its just to prevent AIOOBE on empty terms
+ if ( buffer.length() > 0 && buffer.charAt( buffer.length() - 1 ) == ( 'z' ) ) {
+ buffer.setCharAt( buffer.length() - 1, 'x' );
+ }
+ }
+
+ /**
+ * Removes a particle denotion ("ge") from a term.
+ */
+ private void removeParticleDenotion( StringBuilder buffer )
+ {
+ if ( buffer.length() > 4 ) {
+ for ( int c = 0; c < buffer.length() - 3; c++ ) {
+ if ( buffer.substring( c, c + 4 ).equals( "gege" ) ) {
+ buffer.delete( c, c + 2 );
+ return;
+ }
+ }
+ }
+ }
+
+ /**
+ * Do some substitutions for the term to reduce overstemming:
+ *
+ * - Substitute Umlauts with their corresponding vowel: äöü -> aou,
+ * "ß" is substituted by "ss"
+ * - Substitute a second char of a pair of equal characters with
+ * an asterisk: ?? -> ?*
+ * - Substitute some common character combinations with a token:
+ * sch/ch/ei/ie/ig/st -> $/§/%/&/#/!
+ */
+ private void substitute( StringBuilder buffer )
+ {
+ substCount = 0;
+ for ( int c = 0; c < buffer.length(); c++ ) {
+ // Replace the second char of a pair of the equal characters with an asterisk
+ if ( c > 0 && buffer.charAt( c ) == buffer.charAt ( c - 1 ) ) {
+ buffer.setCharAt( c, '*' );
+ }
+ // Substitute Umlauts.
+ else if ( buffer.charAt( c ) == 'ä' ) {
+ buffer.setCharAt( c, 'a' );
+ }
+ else if ( buffer.charAt( c ) == 'ö' ) {
+ buffer.setCharAt( c, 'o' );
+ }
+ else if ( buffer.charAt( c ) == 'ü' ) {
+ buffer.setCharAt( c, 'u' );
+ }
+ // Fix bug so that 'ß' at the end of a word is replaced.
+ else if ( buffer.charAt( c ) == 'ß' ) {
+ buffer.setCharAt( c, 's' );
+ buffer.insert( c + 1, 's' );
+ substCount++;
+ }
+ // Take care that at least one character is left left side from the current one
+ if ( c < buffer.length() - 1 ) {
+ // Masking several common character combinations with an token
+ if ( ( c < buffer.length() - 2 ) && buffer.charAt( c ) == 's' &&
+ buffer.charAt( c + 1 ) == 'c' && buffer.charAt( c + 2 ) == 'h' )
+ {
+ buffer.setCharAt( c, '$' );
+ buffer.delete( c + 1, c + 3 );
+ substCount =+ 2;
+ }
+ else if ( buffer.charAt( c ) == 'c' && buffer.charAt( c + 1 ) == 'h' ) {
+ buffer.setCharAt( c, '§' );
+ buffer.deleteCharAt( c + 1 );
+ substCount++;
+ }
+ else if ( buffer.charAt( c ) == 'e' && buffer.charAt( c + 1 ) == 'i' ) {
+ buffer.setCharAt( c, '%' );
+ buffer.deleteCharAt( c + 1 );
+ substCount++;
+ }
+ else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'e' ) {
+ buffer.setCharAt( c, '&' );
+ buffer.deleteCharAt( c + 1 );
+ substCount++;
+ }
+ else if ( buffer.charAt( c ) == 'i' && buffer.charAt( c + 1 ) == 'g' ) {
+ buffer.setCharAt( c, '#' );
+ buffer.deleteCharAt( c + 1 );
+ substCount++;
+ }
+ else if ( buffer.charAt( c ) == 's' && buffer.charAt( c + 1 ) == 't' ) {
+ buffer.setCharAt( c, '!' );
+ buffer.deleteCharAt( c + 1 );
+ substCount++;
+ }
+ }
+ }
+ }
+
+ /**
+ * Undoes the changes made by substitute(). That are character pairs and
+ * character combinations. Umlauts will remain as their corresponding vowel,
+ * as "ß" remains as "ss".
+ */
+ private void resubstitute( StringBuilder buffer )
+ {
+ for ( int c = 0; c < buffer.length(); c++ ) {
+ if ( buffer.charAt( c ) == '*' ) {
+ char x = buffer.charAt( c - 1 );
+ buffer.setCharAt( c, x );
+ }
+ else if ( buffer.charAt( c ) == '$' ) {
+ buffer.setCharAt( c, 's' );
+ buffer.insert( c + 1, new char[]{'c', 'h'}, 0, 2 );
+ }
+ else if ( buffer.charAt( c ) == '§' ) {
+ buffer.setCharAt( c, 'c' );
+ buffer.insert( c + 1, 'h' );
+ }
+ else if ( buffer.charAt( c ) == '%' ) {
+ buffer.setCharAt( c, 'e' );
+ buffer.insert( c + 1, 'i' );
+ }
+ else if ( buffer.charAt( c ) == '&' ) {
+ buffer.setCharAt( c, 'i' );
+ buffer.insert( c + 1, 'e' );
+ }
+ else if ( buffer.charAt( c ) == '#' ) {
+ buffer.setCharAt( c, 'i' );
+ buffer.insert( c + 1, 'g' );
+ }
+ else if ( buffer.charAt( c ) == '!' ) {
+ buffer.setCharAt( c, 's' );
+ buffer.insert( c + 1, 't' );
+ }
+ }
+ }
+
+}
fnp / pylucene.git / blobdiff