1 package org.apache.lucene.analysis.standard.std31;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
21 import org.apache.lucene.analysis.standard.StandardTokenizerInterface;
22 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
25 * This class implements UAX29URLEmailTokenizer, except with a bug
26 * (https://issues.apache.org/jira/browse/LUCENE-3358) where Han and Hiragana
27 * characters would be split from combining characters:
28 * @deprecated This class is only for exact backwards compatibility
37 %class UAX29URLEmailTokenizerImpl31
38 %implements StandardTokenizerInterface
39 %function getNextToken
42 %include src/java/org/apache/lucene/analysis/standard/std31/SUPPLEMENTARY.jflex-macro
43 ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
44 Format = ([\p{WB:Format}] | {FormatSupp})
45 Numeric = ([\p{WB:Numeric}] | {NumericSupp})
46 Extend = ([\p{WB:Extend}] | {ExtendSupp})
47 Katakana = ([\p{WB:Katakana}] | {KatakanaSupp})
48 MidLetter = ([\p{WB:MidLetter}] | {MidLetterSupp})
49 MidNum = ([\p{WB:MidNum}] | {MidNumSupp})
50 MidNumLet = ([\p{WB:MidNumLet}] | {MidNumLetSupp})
51 ExtendNumLet = ([\p{WB:ExtendNumLet}] | {ExtendNumLetSupp})
52 ComplexContext = ([\p{LB:Complex_Context}] | {ComplexContextSupp})
53 Han = ([\p{Script:Han}] | {HanSupp})
54 Hiragana = ([\p{Script:Hiragana}] | {HiraganaSupp})
56 // Script=Hangul & Aletter
57 HangulEx = (!(!\p{Script:Hangul}|!\p{WB:ALetter})) ({Format} | {Extend})*
58 // UAX#29 WB4. X (Extend | Format)* --> X
60 ALetterEx = {ALetter} ({Format} | {Extend})*
61 // TODO: Convert hard-coded full-width numeric range to property intersection (something like [\p{Full-Width}&&\p{Numeric}]) once JFlex supports it
62 NumericEx = ({Numeric} | [\uFF10-\uFF19]) ({Format} | {Extend})*
63 KatakanaEx = {Katakana} ({Format} | {Extend})*
64 MidLetterEx = ({MidLetter} | {MidNumLet}) ({Format} | {Extend})*
65 MidNumericEx = ({MidNum} | {MidNumLet}) ({Format} | {Extend})*
66 ExtendNumLetEx = {ExtendNumLet} ({Format} | {Extend})*
69 // URL and E-mail syntax specifications:
71 // RFC-952: DOD INTERNET HOST TABLE SPECIFICATION
72 // RFC-1035: DOMAIN NAMES - IMPLEMENTATION AND SPECIFICATION
73 // RFC-1123: Requirements for Internet Hosts - Application and Support
74 // RFC-1738: Uniform Resource Locators (URL)
75 // RFC-3986: Uniform Resource Identifier (URI): Generic Syntax
76 // RFC-5234: Augmented BNF for Syntax Specifications: ABNF
77 // RFC-5321: Simple Mail Transfer Protocol
78 // RFC-5322: Internet Message Format
80 %include src/java/org/apache/lucene/analysis/standard/std31/ASCIITLD.jflex-macro
82 DomainLabel = [A-Za-z0-9] ([-A-Za-z0-9]* [A-Za-z0-9])?
83 DomainNameStrict = {DomainLabel} ("." {DomainLabel})* {ASCIITLD}
84 DomainNameLoose = {DomainLabel} ("." {DomainLabel})*
86 IPv4DecimalOctet = "0"{0,2} [0-9] | "0"? [1-9][0-9] | "1" [0-9][0-9] | "2" ([0-4][0-9] | "5" [0-5])
87 IPv4Address = {IPv4DecimalOctet} ("." {IPv4DecimalOctet}){3}
88 IPv6Hex16Bit = [0-9A-Fa-f]{1,4}
89 IPv6LeastSignificant32Bits = {IPv4Address} | ({IPv6Hex16Bit} ":" {IPv6Hex16Bit})
90 IPv6Address = ({IPv6Hex16Bit} ":"){6} {IPv6LeastSignificant32Bits}
91 | "::" ({IPv6Hex16Bit} ":"){5} {IPv6LeastSignificant32Bits}
92 | {IPv6Hex16Bit}? "::" ({IPv6Hex16Bit} ":"){4} {IPv6LeastSignificant32Bits}
93 | (({IPv6Hex16Bit} ":"){0,1} {IPv6Hex16Bit})? "::" ({IPv6Hex16Bit} ":"){3} {IPv6LeastSignificant32Bits}
94 | (({IPv6Hex16Bit} ":"){0,2} {IPv6Hex16Bit})? "::" ({IPv6Hex16Bit} ":"){2} {IPv6LeastSignificant32Bits}
95 | (({IPv6Hex16Bit} ":"){0,3} {IPv6Hex16Bit})? "::" {IPv6Hex16Bit} ":" {IPv6LeastSignificant32Bits}
96 | (({IPv6Hex16Bit} ":"){0,4} {IPv6Hex16Bit})? "::" {IPv6LeastSignificant32Bits}
97 | (({IPv6Hex16Bit} ":"){0,5} {IPv6Hex16Bit})? "::" {IPv6Hex16Bit}
98 | (({IPv6Hex16Bit} ":"){0,6} {IPv6Hex16Bit})? "::"
100 URIunreserved = [-._~A-Za-z0-9]
101 URIpercentEncoded = "%" [0-9A-Fa-f]{2}
102 URIsubDelims = [!$&'()*+,;=]
103 URIloginSegment = ({URIunreserved} | {URIpercentEncoded} | {URIsubDelims})*
104 URIlogin = {URIloginSegment} (":" {URIloginSegment})? "@"
105 URIquery = "?" ({URIunreserved} | {URIpercentEncoded} | {URIsubDelims} | [:@/?])*
106 URIfragment = "#" ({URIunreserved} | {URIpercentEncoded} | {URIsubDelims} | [:@/?])*
107 URIport = ":" [0-9]{1,5}
108 URIhostStrict = ("[" {IPv6Address} "]") | {IPv4Address} | {DomainNameStrict}
109 URIhostLoose = ("[" {IPv6Address} "]") | {IPv4Address} | {DomainNameLoose}
111 URIauthorityStrict = {URIhostStrict} {URIport}?
112 URIauthorityLoose = {URIlogin}? {URIhostLoose} {URIport}?
114 HTTPsegment = ({URIunreserved} | {URIpercentEncoded} | [;:@&=])*
115 HTTPpath = ("/" {HTTPsegment})*
116 HTTPscheme = [hH][tT][tT][pP][sS]? "://"
117 HTTPurlFull = {HTTPscheme} {URIauthorityLoose} {HTTPpath}? {URIquery}? {URIfragment}?
118 // {HTTPurlNoScheme} excludes {URIlogin}, because it could otherwise accept e-mail addresses
119 HTTPurlNoScheme = {URIauthorityStrict} {HTTPpath}? {URIquery}? {URIfragment}?
120 HTTPurl = {HTTPurlFull} | {HTTPurlNoScheme}
122 FTPorFILEsegment = ({URIunreserved} | {URIpercentEncoded} | [?:@&=])*
123 FTPorFILEpath = "/" {FTPorFILEsegment} ("/" {FTPorFILEsegment})*
124 FTPtype = ";" [tT][yY][pP][eE] "=" [aAiIdD]
125 FTPscheme = [fF][tT][pP] "://"
126 FTPurl = {FTPscheme} {URIauthorityLoose} {FTPorFILEpath} {FTPtype}? {URIfragment}?
128 FILEscheme = [fF][iI][lL][eE] "://"
129 FILEurl = {FILEscheme} {URIhostLoose}? {FTPorFILEpath} {URIfragment}?
131 URL = {HTTPurl} | {FTPurl} | {FILEurl}
133 EMAILquotedString = [\"] ([\u0001-\u0008\u000B\u000C\u000E-\u0021\u0023-\u005B\u005D-\u007E] | [\\] [\u0000-\u007F])* [\"]
134 EMAILatomText = [A-Za-z0-9!#$%&'*+-/=?\^_`{|}~]
135 EMAILlabel = {EMAILatomText}+ | {EMAILquotedString}
136 EMAILlocalPart = {EMAILlabel} ("." {EMAILlabel})*
137 EMAILdomainLiteralText = [\u0001-\u0008\u000B\u000C\u000E-\u005A\u005E-\u007F] | [\\] [\u0000-\u007F]
138 // DFA minimization allows {IPv6Address} and {IPv4Address} to be included
139 // in the {EMAILbracketedHost} definition without incurring any size penalties,
140 // since {EMAILdomainLiteralText} recognizes all valid IP addresses.
141 // The IP address regexes are included in {EMAILbracketedHost} simply as a
142 // reminder that they are acceptable bracketed host forms.
143 EMAILbracketedHost = "[" ({EMAILdomainLiteralText}* | {IPv4Address} | [iI][pP][vV] "6:" {IPv6Address}) "]"
144 EMAIL = {EMAILlocalPart} "@" ({DomainNameStrict} | {EMAILbracketedHost})
148 /** Alphanumeric sequences */
149 public static final int WORD_TYPE = UAX29URLEmailTokenizer.ALPHANUM;
152 public static final int NUMERIC_TYPE = UAX29URLEmailTokenizer.NUM;
155 * Chars in class \p{Line_Break = Complex_Context} are from South East Asian
156 * scripts (Thai, Lao, Myanmar, Khmer, etc.). Sequences of these are kept
157 * together as as a single token rather than broken up, because the logic
158 * required to break them at word boundaries is too complex for UAX#29.
160 * See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA
162 public static final int SOUTH_EAST_ASIAN_TYPE = UAX29URLEmailTokenizer.SOUTHEAST_ASIAN;
164 public static final int IDEOGRAPHIC_TYPE = UAX29URLEmailTokenizer.IDEOGRAPHIC;
166 public static final int HIRAGANA_TYPE = UAX29URLEmailTokenizer.HIRAGANA;
168 public static final int KATAKANA_TYPE = UAX29URLEmailTokenizer.KATAKANA;
170 public static final int HANGUL_TYPE = UAX29URLEmailTokenizer.HANGUL;
172 public static final int EMAIL_TYPE = UAX29URLEmailTokenizer.EMAIL;
174 public static final int URL_TYPE = UAX29URLEmailTokenizer.URL;
176 public final int yychar()
182 * Fills CharTermAttribute with the current token text.
184 public final void getText(CharTermAttribute t) {
185 t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
194 <<EOF>> { return StandardTokenizerInterface.YYEOF; }
196 {URL} { return URL_TYPE; }
197 {EMAIL} { return EMAIL_TYPE; }
199 // UAX#29 WB8. Numeric × Numeric
200 // WB11. Numeric (MidNum | MidNumLet) × Numeric
201 // WB12. Numeric × (MidNum | MidNumLet) Numeric
202 // WB13a. (ALetter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet
203 // WB13b. ExtendNumLet × (ALetter | Numeric | Katakana)
205 {ExtendNumLetEx}* {NumericEx} ({ExtendNumLetEx}+ {NumericEx}
206 | {MidNumericEx} {NumericEx}
209 { return NUMERIC_TYPE; }
211 // subset of the below for typing purposes only!
213 { return HANGUL_TYPE; }
216 { return KATAKANA_TYPE; }
218 // UAX#29 WB5. ALetter × ALetter
219 // WB6. ALetter × (MidLetter | MidNumLet) ALetter
220 // WB7. ALetter (MidLetter | MidNumLet) × ALetter
221 // WB9. ALetter × Numeric
222 // WB10. Numeric × ALetter
223 // WB13. Katakana × Katakana
224 // WB13a. (ALetter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet
225 // WB13b. ExtendNumLet × (ALetter | Numeric | Katakana)
227 {ExtendNumLetEx}* ( {KatakanaEx} ({ExtendNumLetEx}* {KatakanaEx})*
228 | ( {NumericEx} ({ExtendNumLetEx}+ {NumericEx} | {MidNumericEx} {NumericEx} | {NumericEx})*
229 | {ALetterEx} ({ExtendNumLetEx}+ {ALetterEx} | {MidLetterEx} {ALetterEx} | {ALetterEx})* )+ )
230 ({ExtendNumLetEx}+ ( {KatakanaEx} ({ExtendNumLetEx}* {KatakanaEx})*
231 | ( {NumericEx} ({ExtendNumLetEx}+ {NumericEx} | {MidNumericEx} {NumericEx} | {NumericEx})*
232 | {ALetterEx} ({ExtendNumLetEx}+ {ALetterEx} | {MidLetterEx} {ALetterEx} | {ALetterEx})* )+ ) )*
234 { return WORD_TYPE; }
239 // [C]haracters with the Line_Break property values of Contingent_Break (CB),
240 // Complex_Context (SA/South East Asian), and XX (Unknown) are assigned word
241 // boundary property values based on criteria outside of the scope of this
242 // annex. That means that satisfactory treatment of languages like Chinese
243 // or Thai requires special handling.
245 // In Unicode 6.0, only one character has the \p{Line_Break = Contingent_Break}
246 // property: U+FFFC (  ) OBJECT REPLACEMENT CHARACTER.
248 // In the ICU implementation of UAX#29, \p{Line_Break = Complex_Context}
249 // character sequences (from South East Asian scripts like Thai, Myanmar, Khmer,
250 // Lao, etc.) are kept together. This grammar does the same below.
252 // See also the Unicode Line Breaking Algorithm:
254 // http://www.unicode.org/reports/tr14/#SA
256 {ComplexContext}+ { return SOUTH_EAST_ASIAN_TYPE; }
258 // UAX#29 WB14. Any ÷ Any
260 {Han} { return IDEOGRAPHIC_TYPE; }
261 {Hiragana} { return HIRAGANA_TYPE; }
264 // UAX#29 WB3. CR × LF
265 // WB3a. (Newline | CR | LF) ÷
266 // WB3b. ÷ (Newline | CR | LF)
269 [^] { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }