2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
19 package org.apache.lucene.analysis.wikipedia;
21 import java.io.StringReader;
22 import java.io.IOException;
24 import java.util.HashSet;
26 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
27 import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
29 import static org.apache.lucene.analysis.wikipedia.WikipediaTokenizer.*;
32 * Basic Tests for {@link WikipediaTokenizer}
34 public class WikipediaTokenizerTest extends BaseTokenStreamTestCase {
35 protected static final String LINK_PHRASES = "click [[link here again]] click [http://lucene.apache.org here again] [[Category:a b c d]]";
37 public void testSimple() throws Exception {
38 String text = "This is a [[Category:foo]]";
39 WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(text));
40 assertTokenStreamContents(tf,
41 new String[] { "This", "is", "a", "foo" },
42 new int[] { 0, 5, 8, 21 },
43 new int[] { 4, 7, 9, 24 },
44 new String[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", CATEGORY },
45 new int[] { 1, 1, 1, 1, },
49 public void testHandwritten() throws Exception {
50 // make sure all tokens are in only one type
51 String test = "[[link]] This is a [[Category:foo]] Category This is a linked [[:Category:bar none withstanding]] "
52 + "Category This is (parens) This is a [[link]] This is an external URL [http://lucene.apache.org] "
53 + "Here is ''italics'' and ''more italics'', '''bold''' and '''''five quotes''''' "
54 + " This is a [[link|display info]] This is a period. Here is $3.25 and here is 3.50. Here's Johnny. "
55 + "==heading== ===sub head=== followed by some text [[Category:blah| ]] "
56 + "''[[Category:ital_cat]]'' here is some that is ''italics [[Category:foo]] but is never closed."
57 + "'''same [[Category:foo]] goes for this '''''and2 [[Category:foo]] and this"
58 + " [http://foo.boo.com/test/test/ Test Test] [http://foo.boo.com/test/test/test.html Test Test]"
59 + " [http://foo.boo.com/test/test/test.html?g=b&c=d Test Test] <ref>Citation</ref> <sup>martian</sup> <span class=\"glue\">code</span>";
61 WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test));
62 assertTokenStreamContents(tf,
63 new String[] {"link", "This", "is", "a",
64 "foo", "Category", "This", "is", "a", "linked", "bar", "none",
65 "withstanding", "Category", "This", "is", "parens", "This", "is", "a",
66 "link", "This", "is", "an", "external", "URL",
67 "http://lucene.apache.org", "Here", "is", "italics", "and", "more",
68 "italics", "bold", "and", "five", "quotes", "This", "is", "a", "link",
69 "display", "info", "This", "is", "a", "period", "Here", "is", "3.25",
70 "and", "here", "is", "3.50", "Here's", "Johnny", "heading", "sub",
71 "head", "followed", "by", "some", "text", "blah", "ital", "cat",
72 "here", "is", "some", "that", "is", "italics", "foo", "but", "is",
73 "never", "closed", "same", "foo", "goes", "for", "this", "and2", "foo",
74 "and", "this", "http://foo.boo.com/test/test/", "Test", "Test",
75 "http://foo.boo.com/test/test/test.html", "Test", "Test",
76 "http://foo.boo.com/test/test/test.html?g=b&c=d", "Test", "Test",
77 "Citation", "martian", "code"},
78 new String[] {INTERNAL_LINK,
79 "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", CATEGORY, "<ALPHANUM>",
80 "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", CATEGORY,
81 CATEGORY, CATEGORY, "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
82 "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", INTERNAL_LINK,
83 "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
84 EXTERNAL_LINK_URL, "<ALPHANUM>", "<ALPHANUM>", ITALICS, "<ALPHANUM>",
85 ITALICS, ITALICS, BOLD, "<ALPHANUM>", BOLD_ITALICS, BOLD_ITALICS,
86 "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", INTERNAL_LINK, INTERNAL_LINK,
87 INTERNAL_LINK, "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
88 "<ALPHANUM>", "<ALPHANUM>", "<NUM>", "<ALPHANUM>", "<ALPHANUM>",
89 "<ALPHANUM>", "<NUM>", "<APOSTROPHE>", "<ALPHANUM>", HEADING,
90 SUB_HEADING, SUB_HEADING, "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
91 "<ALPHANUM>", CATEGORY, CATEGORY, CATEGORY, "<ALPHANUM>", "<ALPHANUM>",
92 "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", ITALICS, CATEGORY,
93 "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", BOLD, CATEGORY,
94 "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", BOLD_ITALICS, CATEGORY,
95 "<ALPHANUM>", "<ALPHANUM>", EXTERNAL_LINK_URL, EXTERNAL_LINK,
96 EXTERNAL_LINK, EXTERNAL_LINK_URL, EXTERNAL_LINK, EXTERNAL_LINK,
97 EXTERNAL_LINK_URL, EXTERNAL_LINK, EXTERNAL_LINK, CITATION,
98 "<ALPHANUM>", "<ALPHANUM>"});
101 public void testLinkPhrases() throws Exception {
102 WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(LINK_PHRASES));
103 checkLinkPhrases(tf);
106 private void checkLinkPhrases(WikipediaTokenizer tf) throws IOException {
107 assertTokenStreamContents(tf,
108 new String[] { "click", "link", "here", "again", "click",
109 "http://lucene.apache.org", "here", "again", "a", "b", "c", "d" },
110 new int[] { 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1 });
113 public void testLinks() throws Exception {
114 String test = "[http://lucene.apache.org/java/docs/index.html#news here] [http://lucene.apache.org/java/docs/index.html?b=c here] [https://lucene.apache.org/java/docs/index.html?b=c here]";
115 WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test));
116 assertTokenStreamContents(tf,
117 new String[] { "http://lucene.apache.org/java/docs/index.html#news", "here",
118 "http://lucene.apache.org/java/docs/index.html?b=c", "here",
119 "https://lucene.apache.org/java/docs/index.html?b=c", "here" },
120 new String[] { EXTERNAL_LINK_URL, EXTERNAL_LINK,
121 EXTERNAL_LINK_URL, EXTERNAL_LINK,
122 EXTERNAL_LINK_URL, EXTERNAL_LINK, });
125 public void testLucene1133() throws Exception {
126 Set<String> untoks = new HashSet<String>();
127 untoks.add(WikipediaTokenizer.CATEGORY);
128 untoks.add(WikipediaTokenizer.ITALICS);
129 //should be exactly the same, regardless of untoks
130 WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(LINK_PHRASES), WikipediaTokenizer.TOKENS_ONLY, untoks);
131 checkLinkPhrases(tf);
132 String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
133 tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.UNTOKENIZED_ONLY, untoks);
134 assertTokenStreamContents(tf,
135 new String[] { "a b c d", "e f g", "link", "here", "link",
136 "there", "italics here", "something", "more italics", "h i j" },
137 new int[] { 11, 32, 42, 47, 56, 61, 71, 86, 98, 124 },
138 new int[] { 18, 37, 46, 51, 60, 66, 83, 95, 110, 133 },
139 new int[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }
143 public void testBoth() throws Exception {
144 Set<String> untoks = new HashSet<String>();
145 untoks.add(WikipediaTokenizer.CATEGORY);
146 untoks.add(WikipediaTokenizer.ITALICS);
147 String test = "[[Category:a b c d]] [[Category:e f g]] [[link here]] [[link there]] ''italics here'' something ''more italics'' [[Category:h i j]]";
148 //should output all the indivual tokens plus the untokenized tokens as well. Untokenized tokens
149 WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.BOTH, untoks);
150 assertTokenStreamContents(tf,
151 new String[] { "a b c d", "a", "b", "c", "d", "e f g", "e", "f", "g",
152 "link", "here", "link", "there", "italics here", "italics", "here",
153 "something", "more italics", "more", "italics", "h i j", "h", "i", "j" },
154 new int[] { 11, 11, 13, 15, 17, 32, 32, 34, 36, 42, 47, 56, 61, 71, 71, 79, 86, 98, 98, 103, 124, 124, 128, 132 },
155 new int[] { 18, 12, 14, 16, 18, 37, 33, 35, 37, 46, 51, 60, 66, 83, 78, 83, 95, 110, 102, 110, 133, 125, 129, 133 },
156 new int[] { 1, 0, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, 1, 1 }
159 // now check the flags, TODO: add way to check flags from BaseTokenStreamTestCase?
160 tf = new WikipediaTokenizer(new StringReader(test), WikipediaTokenizer.BOTH, untoks);
161 int expectedFlags[] = new int[] { UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, 0,
162 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, UNTOKENIZED_TOKEN_FLAG, 0, 0, 0 };
163 FlagsAttribute flagsAtt = tf.addAttribute(FlagsAttribute.class);
165 for (int i = 0; i < expectedFlags.length; i++) {
166 assertTrue(tf.incrementToken());
167 assertEquals("flags " + i, expectedFlags[i], flagsAtt.getFlags());
169 assertFalse(tf.incrementToken());