--- /dev/null
+package org.apache.lucene.analysis.nl;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharArraySet;
+import org.apache.lucene.util.Version;
+
+/**
+ * Test the Dutch Stem Filter, which only modifies the term text.
+ *
+ * The code states that it uses the snowball algorithm, but tests reveal some differences.
+ *
+ */
+public class TestDutchStemmer extends BaseTokenStreamTestCase {
+
+ public void testWithSnowballExamples() throws Exception {
+ check("lichaamsziek", "lichaamsziek");
+ check("lichamelijk", "licham");
+ check("lichamelijke", "licham");
+ check("lichamelijkheden", "licham");
+ check("lichamen", "licham");
+ check("lichere", "licher");
+ check("licht", "licht");
+ check("lichtbeeld", "lichtbeeld");
+ check("lichtbruin", "lichtbruin");
+ check("lichtdoorlatende", "lichtdoorlat");
+ check("lichte", "licht");
+ check("lichten", "licht");
+ check("lichtende", "lichtend");
+ check("lichtenvoorde", "lichtenvoord");
+ check("lichter", "lichter");
+ check("lichtere", "lichter");
+ check("lichters", "lichter");
+ check("lichtgevoeligheid", "lichtgevoel");
+ check("lichtgewicht", "lichtgewicht");
+ check("lichtgrijs", "lichtgrijs");
+ check("lichthoeveelheid", "lichthoevel");
+ check("lichtintensiteit", "lichtintensiteit");
+ check("lichtje", "lichtj");
+ check("lichtjes", "lichtjes");
+ check("lichtkranten", "lichtkrant");
+ check("lichtkring", "lichtkring");
+ check("lichtkringen", "lichtkring");
+ check("lichtregelsystemen", "lichtregelsystem");
+ check("lichtste", "lichtst");
+ check("lichtstromende", "lichtstrom");
+ check("lichtte", "licht");
+ check("lichtten", "licht");
+ check("lichttoetreding", "lichttoetred");
+ check("lichtverontreinigde", "lichtverontreinigd");
+ check("lichtzinnige", "lichtzinn");
+ check("lid", "lid");
+ check("lidia", "lidia");
+ check("lidmaatschap", "lidmaatschap");
+ check("lidstaten", "lidstat");
+ check("lidvereniging", "lidveren");
+ check("opgingen", "opging");
+ check("opglanzing", "opglanz");
+ check("opglanzingen", "opglanz");
+ check("opglimlachten", "opglimlacht");
+ check("opglimpen", "opglimp");
+ check("opglimpende", "opglimp");
+ check("opglimping", "opglimp");
+ check("opglimpingen", "opglimp");
+ check("opgraven", "opgrav");
+ check("opgrijnzen", "opgrijnz");
+ check("opgrijzende", "opgrijz");
+ check("opgroeien", "opgroei");
+ check("opgroeiende", "opgroei");
+ check("opgroeiplaats", "opgroeiplat");
+ check("ophaal", "ophal");
+ check("ophaaldienst", "ophaaldienst");
+ check("ophaalkosten", "ophaalkost");
+ check("ophaalsystemen", "ophaalsystem");
+ check("ophaalt", "ophaalt");
+ check("ophaaltruck", "ophaaltruck");
+ check("ophalen", "ophal");
+ check("ophalend", "ophal");
+ check("ophalers", "ophaler");
+ check("ophef", "ophef");
+ check("opheldering", "ophelder");
+ check("ophemelde", "ophemeld");
+ check("ophemelen", "ophemel");
+ check("opheusden", "opheusd");
+ check("ophief", "ophief");
+ check("ophield", "ophield");
+ check("ophieven", "ophiev");
+ check("ophoepelt", "ophoepelt");
+ check("ophoog", "ophog");
+ check("ophoogzand", "ophoogzand");
+ check("ophopen", "ophop");
+ check("ophoping", "ophop");
+ check("ophouden", "ophoud");
+ }
+
+ /**
+ * @deprecated remove this test in Lucene 4.0
+ */
+ @Deprecated
+ public void testOldBuggyStemmer() throws Exception {
+ Analyzer a = new DutchAnalyzer(Version.LUCENE_30);
+ checkOneTermReuse(a, "opheffen", "ophef"); // versus snowball 'opheff'
+ checkOneTermReuse(a, "opheffende", "ophef"); // versus snowball 'opheff'
+ checkOneTermReuse(a, "opheffing", "ophef"); // versus snowball 'opheff'
+ }
+
+ public void testSnowballCorrectness() throws Exception {
+ Analyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT);
+ checkOneTermReuse(a, "opheffen", "opheff");
+ checkOneTermReuse(a, "opheffende", "opheff");
+ checkOneTermReuse(a, "opheffing", "opheff");
+ }
+
+ public void testReusableTokenStream() throws Exception {
+ Analyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT);
+ checkOneTermReuse(a, "lichaamsziek", "lichaamsziek");
+ checkOneTermReuse(a, "lichamelijk", "licham");
+ checkOneTermReuse(a, "lichamelijke", "licham");
+ checkOneTermReuse(a, "lichamelijkheden", "licham");
+ }
+
+ /*
+ * Test that changes to the exclusion table are applied immediately
+ * when using reusable token streams.
+ */
+ public void testExclusionTableReuse() throws Exception {
+ DutchAnalyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT);
+ checkOneTermReuse(a, "lichamelijk", "licham");
+ a.setStemExclusionTable(new String[] { "lichamelijk" });
+ checkOneTermReuse(a, "lichamelijk", "lichamelijk");
+
+
+ }
+
+ public void testExclusionTableViaCtor() throws IOException {
+ CharArraySet set = new CharArraySet(Version.LUCENE_30, 1, true);
+ set.add("lichamelijk");
+ DutchAnalyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET, set);
+ assertAnalyzesToReuse(a, "lichamelijk lichamelijke", new String[] { "lichamelijk", "licham" });
+
+ a = new DutchAnalyzer(TEST_VERSION_CURRENT, CharArraySet.EMPTY_SET, set);
+ assertAnalyzesTo(a, "lichamelijk lichamelijke", new String[] { "lichamelijk", "licham" });
+
+ }
+
+ /*
+ * Test that changes to the dictionary stemming table are applied immediately
+ * when using reusable token streams.
+ */
+ public void testStemDictionaryReuse() throws Exception {
+ DutchAnalyzer a = new DutchAnalyzer(TEST_VERSION_CURRENT);
+ checkOneTermReuse(a, "lichamelijk", "licham");
+ File customDictFile = getDataFile("customStemDict.txt");
+ a.setStemDictionary(customDictFile);
+ checkOneTermReuse(a, "lichamelijk", "somethingentirelydifferent");
+ }
+
+ /**
+ * Prior to 3.1, this analyzer had no lowercase filter.
+ * stopwords were case sensitive. Preserve this for back compat.
+ * @deprecated Remove this test in Lucene 4.0
+ */
+ @Deprecated
+ public void testBuggyStopwordsCasing() throws IOException {
+ DutchAnalyzer a = new DutchAnalyzer(Version.LUCENE_30);
+ assertAnalyzesTo(a, "Zelf", new String[] { "zelf" });
+ }
+
+ /**
+ * Test that stopwords are not case sensitive
+ */
+ public void testStopwordsCasing() throws IOException {
+ DutchAnalyzer a = new DutchAnalyzer(Version.LUCENE_31);
+ assertAnalyzesTo(a, "Zelf", new String[] { });
+ }
+
+ private void check(final String input, final String expected) throws Exception {
+ checkOneTerm(new DutchAnalyzer(TEST_VERSION_CURRENT), input, expected);
+ }
+
+ /** blast some random strings through the analyzer */
+ public void testRandomStrings() throws Exception {
+ checkRandomData(random, new DutchAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
+ }
+
+}
\ No newline at end of file
fnp / pylucene.git / blobdiff