+++ /dev/null
-package org.apache.lucene.analysis.th;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.StringReader;
-
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
-import org.apache.lucene.util.Version;
-
-/**
- * Test case for ThaiAnalyzer, modified from TestFrenchAnalyzer
- *
- * @version 0.1
- */
-
-public class TestThaiAnalyzer extends BaseTokenStreamTestCase {
-
- /*
- * testcase for offsets
- */
- public void testOffsets() throws Exception {
- assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
- assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "การที่ได้ต้องแสดงว่างานดี",
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี" },
- new int[] { 0, 3, 6, 9, 13, 17, 20, 23 },
- new int[] { 3, 6, 9, 13, 17, 20, 23, 25 });
- }
-
- public void testTokenType() throws Exception {
- assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
- assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "การที่ได้ต้องแสดงว่างานดี ๑๒๓",
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี", "๑๒๓" },
- new String[] { "<SOUTHEAST_ASIAN>", "<SOUTHEAST_ASIAN>",
- "<SOUTHEAST_ASIAN>", "<SOUTHEAST_ASIAN>",
- "<SOUTHEAST_ASIAN>", "<SOUTHEAST_ASIAN>",
- "<SOUTHEAST_ASIAN>", "<SOUTHEAST_ASIAN>",
- "<NUM>" });
- }
-
- /**
- * Thai numeric tokens were typed as <ALPHANUM> instead of <NUM>.
- * @deprecated testing backwards behavior
- */
- @Deprecated
- public void testBuggyTokenType30() throws Exception {
- assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
- assertAnalyzesTo(new ThaiAnalyzer(Version.LUCENE_30), "การที่ได้ต้องแสดงว่างานดี ๑๒๓",
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี", "๑๒๓" },
- new String[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
- "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
- "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>" });
- }
-
- /** @deprecated testing backwards behavior */
- @Deprecated
- public void testAnalyzer30() throws Exception {
- assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
- ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_30);
-
- assertAnalyzesTo(analyzer, "", new String[] {});
-
- assertAnalyzesTo(
- analyzer,
- "การที่ได้ต้องแสดงว่างานดี",
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี"});
-
- assertAnalyzesTo(
- analyzer,
- "บริษัทชื่อ XY&Z - คุยกับ xyz@demo.com",
- new String[] { "บริษัท", "ชื่อ", "xy&z", "คุย", "กับ", "xyz@demo.com" });
-
- // English stop words
- assertAnalyzesTo(
- analyzer,
- "ประโยคว่า The quick brown fox jumped over the lazy dogs",
- new String[] { "ประโยค", "ว่า", "quick", "brown", "fox", "jumped", "over", "lazy", "dogs" });
- }
-
- /*
- * Test that position increments are adjusted correctly for stopwords.
- */
- public void testPositionIncrements() throws Exception {
- assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
- ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT);
-
- assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "การที่ได้ต้อง the แสดงว่างานดี",
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี" },
- new int[] { 0, 3, 6, 9, 18, 22, 25, 28 },
- new int[] { 3, 6, 9, 13, 22, 25, 28, 30 },
- new int[] { 1, 1, 1, 1, 2, 1, 1, 1 });
-
- // case that a stopword is adjacent to thai text, with no whitespace
- assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "การที่ได้ต้องthe แสดงว่างานดี",
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี" },
- new int[] { 0, 3, 6, 9, 17, 21, 24, 27 },
- new int[] { 3, 6, 9, 13, 21, 24, 27, 29 },
- new int[] { 1, 1, 1, 1, 2, 1, 1, 1 });
- }
-
- public void testReusableTokenStream() throws Exception {
- assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
- ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT);
- assertAnalyzesToReuse(analyzer, "", new String[] {});
-
- assertAnalyzesToReuse(
- analyzer,
- "การที่ได้ต้องแสดงว่างานดี",
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี"});
-
- assertAnalyzesToReuse(
- analyzer,
- "บริษัทชื่อ XY&Z - คุยกับ xyz@demo.com",
- new String[] { "บริษัท", "ชื่อ", "xy", "z", "คุย", "กับ", "xyz", "demo.com" });
- }
-
- /** @deprecated, for version back compat */
- @Deprecated
- public void testReusableTokenStream30() throws Exception {
- assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
- ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_30);
- assertAnalyzesToReuse(analyzer, "", new String[] {});
-
- assertAnalyzesToReuse(
- analyzer,
- "การที่ได้ต้องแสดงว่างานดี",
- new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี"});
-
- assertAnalyzesToReuse(
- analyzer,
- "บริษัทชื่อ XY&Z - คุยกับ xyz@demo.com",
- new String[] { "บริษัท", "ชื่อ", "xy&z", "คุย", "กับ", "xyz@demo.com" });
- }
-
- /** blast some random strings through the analyzer */
- public void testRandomStrings() throws Exception {
- checkRandomData(random, new ThaiAnalyzer(TEST_VERSION_CURRENT), 10000*RANDOM_MULTIPLIER);
- }
-
- // LUCENE-3044
- public void testAttributeReuse() throws Exception {
- assumeTrue("JRE does not support Thai dictionary-based BreakIterator", ThaiWordFilter.DBBI_AVAILABLE);
- ThaiAnalyzer analyzer = new ThaiAnalyzer(Version.LUCENE_30);
- // just consume
- TokenStream ts = analyzer.reusableTokenStream("dummy", new StringReader("ภาษาไทย"));
- assertTokenStreamContents(ts, new String[] { "ภาษา", "ไทย" });
- // this consumer adds flagsAtt, which this analyzer does not use.
- ts = analyzer.reusableTokenStream("dummy", new StringReader("ภาษาไทย"));
- ts.addAttribute(FlagsAttribute.class);
- assertTokenStreamContents(ts, new String[] { "ภาษา", "ไทย" });
- }
-}
fnp / pylucene.git / blobdiff