lucene-java-3.4.0/lucene/src/java/org/apache/lucene/analysis/standard/std31/UAX29URLEmailTokenizerImpl31.jflex

   1 package org.apache.lucene.analysis.standard.std31;
   2
   3 /**
   4  * Licensed to the Apache Software Foundation (ASF) under one or more
   5  * contributor license agreements.  See the NOTICE file distributed with
   6  * this work for additional information regarding copyright ownership.
   7  * The ASF licenses this file to You under the Apache License, Version 2.0
   8  * (the "License"); you may not use this file except in compliance with
   9  * the License.  You may obtain a copy of the License at
  10  *
  11  *     http://www.apache.org/licenses/LICENSE-2.0
  12  *
  13  * Unless required by applicable law or agreed to in writing, software
  14  * distributed under the License is distributed on an "AS IS" BASIS,
  15  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  16  * See the License for the specific language governing permissions and
  17  * limitations under the License.
  18  */
  19
  20 import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer;
  21 import org.apache.lucene.analysis.standard.StandardTokenizerInterface;
  22 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
  23
  24 /**
  25  * This class implements UAX29URLEmailTokenizer, except with a bug
  26  * (https://issues.apache.org/jira/browse/LUCENE-3358) where Han and Hiragana
  27  * characters would be split from combining characters:
  28  * @deprecated This class is only for exact backwards compatibility
  29  */
  30  @Deprecated
  31 %%
  32
  33 %unicode 6.0
  34 %integer
  35 %final
  36 %public
  37 %class UAX29URLEmailTokenizerImpl31
  38 %implements StandardTokenizerInterface
  39 %function getNextToken
  40 %char
  41
  42 %include src/java/org/apache/lucene/analysis/standard/std31/SUPPLEMENTARY.jflex-macro
  43 ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
  44 Format =  ([\p{WB:Format}] | {FormatSupp})
  45 Numeric = ([\p{WB:Numeric}] | {NumericSupp})
  46 Extend =  ([\p{WB:Extend}] | {ExtendSupp})
  47 Katakana = ([\p{WB:Katakana}] | {KatakanaSupp})
  48 MidLetter = ([\p{WB:MidLetter}] | {MidLetterSupp})
  49 MidNum = ([\p{WB:MidNum}] | {MidNumSupp})
  50 MidNumLet = ([\p{WB:MidNumLet}] | {MidNumLetSupp})
  51 ExtendNumLet = ([\p{WB:ExtendNumLet}] | {ExtendNumLetSupp})
  52 ComplexContext = ([\p{LB:Complex_Context}] | {ComplexContextSupp})
  53 Han = ([\p{Script:Han}] | {HanSupp})
  54 Hiragana = ([\p{Script:Hiragana}] | {HiraganaSupp})
  55
  56 // Script=Hangul & Aletter
  57 HangulEx       = (!(!\p{Script:Hangul}|!\p{WB:ALetter})) ({Format} | {Extend})*
  58 // UAX#29 WB4. X (Extend | Format)* --> X
  59 //
  60 ALetterEx      = {ALetter}                     ({Format} | {Extend})*
  61 // TODO: Convert hard-coded full-width numeric range to property intersection (something like [\p{Full-Width}&&\p{Numeric}]) once JFlex supports it
  62 NumericEx      = ({Numeric} | [\uFF10-\uFF19]) ({Format} | {Extend})*
  63 KatakanaEx     = {Katakana}                    ({Format} | {Extend})*
  64 MidLetterEx    = ({MidLetter} | {MidNumLet})   ({Format} | {Extend})*
  65 MidNumericEx   = ({MidNum} | {MidNumLet})      ({Format} | {Extend})*
  66 ExtendNumLetEx = {ExtendNumLet}                ({Format} | {Extend})*
  67
  68
  69 // URL and E-mail syntax specifications:
  70 //
  71 //     RFC-952:  DOD INTERNET HOST TABLE SPECIFICATION
  72 //     RFC-1035: DOMAIN NAMES - IMPLEMENTATION AND SPECIFICATION
  73 //     RFC-1123: Requirements for Internet Hosts - Application and Support
  74 //     RFC-1738: Uniform Resource Locators (URL)
  75 //     RFC-3986: Uniform Resource Identifier (URI): Generic Syntax
  76 //     RFC-5234: Augmented BNF for Syntax Specifications: ABNF
  77 //     RFC-5321: Simple Mail Transfer Protocol
  78 //     RFC-5322: Internet Message Format
  79
  80 %include src/java/org/apache/lucene/analysis/standard/std31/ASCIITLD.jflex-macro
  81
  82 DomainLabel = [A-Za-z0-9] ([-A-Za-z0-9]* [A-Za-z0-9])?
  83 DomainNameStrict = {DomainLabel} ("." {DomainLabel})* {ASCIITLD}
  84 DomainNameLoose  = {DomainLabel} ("." {DomainLabel})*
  85
  86 IPv4DecimalOctet = "0"{0,2} [0-9] | "0"? [1-9][0-9] | "1" [0-9][0-9] | "2" ([0-4][0-9] | "5" [0-5])
  87 IPv4Address  = {IPv4DecimalOctet} ("." {IPv4DecimalOctet}){3}
  88 IPv6Hex16Bit = [0-9A-Fa-f]{1,4}
  89 IPv6LeastSignificant32Bits = {IPv4Address} | ({IPv6Hex16Bit} ":" {IPv6Hex16Bit})
  90 IPv6Address =                                                  ({IPv6Hex16Bit} ":"){6} {IPv6LeastSignificant32Bits}
  91             |                                             "::" ({IPv6Hex16Bit} ":"){5} {IPv6LeastSignificant32Bits}
  92             |                            {IPv6Hex16Bit}?  "::" ({IPv6Hex16Bit} ":"){4} {IPv6LeastSignificant32Bits}
  93             | (({IPv6Hex16Bit} ":"){0,1} {IPv6Hex16Bit})? "::" ({IPv6Hex16Bit} ":"){3} {IPv6LeastSignificant32Bits}
  94             | (({IPv6Hex16Bit} ":"){0,2} {IPv6Hex16Bit})? "::" ({IPv6Hex16Bit} ":"){2} {IPv6LeastSignificant32Bits}
  95             | (({IPv6Hex16Bit} ":"){0,3} {IPv6Hex16Bit})? "::"  {IPv6Hex16Bit} ":"     {IPv6LeastSignificant32Bits}
  96             | (({IPv6Hex16Bit} ":"){0,4} {IPv6Hex16Bit})? "::"                         {IPv6LeastSignificant32Bits}
  97             | (({IPv6Hex16Bit} ":"){0,5} {IPv6Hex16Bit})? "::"                         {IPv6Hex16Bit}
  98             | (({IPv6Hex16Bit} ":"){0,6} {IPv6Hex16Bit})? "::"
  99
 100 URIunreserved = [-._~A-Za-z0-9]
 101 URIpercentEncoded = "%" [0-9A-Fa-f]{2}
 102 URIsubDelims = [!$&'()*+,;=]
 103 URIloginSegment = ({URIunreserved} | {URIpercentEncoded} | {URIsubDelims})*
 104 URIlogin = {URIloginSegment} (":" {URIloginSegment})? "@"
 105 URIquery    = "?" ({URIunreserved} | {URIpercentEncoded} | {URIsubDelims} | [:@/?])*
 106 URIfragment = "#" ({URIunreserved} | {URIpercentEncoded} | {URIsubDelims} | [:@/?])*
 107 URIport = ":" [0-9]{1,5}
 108 URIhostStrict = ("[" {IPv6Address} "]") | {IPv4Address} | {DomainNameStrict}
 109 URIhostLoose  = ("[" {IPv6Address} "]") | {IPv4Address} | {DomainNameLoose}
 110
 111 URIauthorityStrict =             {URIhostStrict} {URIport}?
 112 URIauthorityLoose  = {URIlogin}? {URIhostLoose}  {URIport}?
 113
 114 HTTPsegment = ({URIunreserved} | {URIpercentEncoded} | [;:@&=])*
 115 HTTPpath = ("/" {HTTPsegment})*
 116 HTTPscheme = [hH][tT][tT][pP][sS]? "://"
 117 HTTPurlFull = {HTTPscheme} {URIauthorityLoose}  {HTTPpath}? {URIquery}? {URIfragment}?
 118 // {HTTPurlNoScheme} excludes {URIlogin}, because it could otherwise accept e-mail addresses
 119 HTTPurlNoScheme =          {URIauthorityStrict} {HTTPpath}? {URIquery}? {URIfragment}?
 120 HTTPurl = {HTTPurlFull} | {HTTPurlNoScheme}
 121
 122 FTPorFILEsegment = ({URIunreserved} | {URIpercentEncoded} | [?:@&=])*
 123 FTPorFILEpath = "/" {FTPorFILEsegment} ("/" {FTPorFILEsegment})*
 124 FTPtype = ";" [tT][yY][pP][eE] "=" [aAiIdD]
 125 FTPscheme = [fF][tT][pP] "://"
 126 FTPurl = {FTPscheme} {URIauthorityLoose} {FTPorFILEpath} {FTPtype}? {URIfragment}?
 127
 128 FILEscheme = [fF][iI][lL][eE] "://"
 129 FILEurl = {FILEscheme} {URIhostLoose}? {FTPorFILEpath} {URIfragment}?
 130
 131 URL = {HTTPurl} | {FTPurl} | {FILEurl}
 132
 133 EMAILquotedString = [\"] ([\u0001-\u0008\u000B\u000C\u000E-\u0021\u0023-\u005B\u005D-\u007E] | [\\] [\u0000-\u007F])* [\"]
 134 EMAILatomText = [A-Za-z0-9!#$%&'*+-/=?\^_`{|}~]
 135 EMAILlabel = {EMAILatomText}+ | {EMAILquotedString}
 136 EMAILlocalPart = {EMAILlabel} ("." {EMAILlabel})*
 137 EMAILdomainLiteralText = [\u0001-\u0008\u000B\u000C\u000E-\u005A\u005E-\u007F] | [\\] [\u0000-\u007F]
 138 // DFA minimization allows {IPv6Address} and {IPv4Address} to be included
 139 // in the {EMAILbracketedHost} definition without incurring any size penalties,
 140 // since {EMAILdomainLiteralText} recognizes all valid IP addresses.
 141 // The IP address regexes are included in {EMAILbracketedHost} simply as a
 142 // reminder that they are acceptable bracketed host forms.
 143 EMAILbracketedHost = "[" ({EMAILdomainLiteralText}* | {IPv4Address} | [iI][pP][vV] "6:" {IPv6Address}) "]"
 144 EMAIL = {EMAILlocalPart} "@" ({DomainNameStrict} | {EMAILbracketedHost})
 145
 146
 147 %{
 148   /** Alphanumeric sequences */
 149   public static final int WORD_TYPE = UAX29URLEmailTokenizer.ALPHANUM;
 150
 151   /** Numbers */
 152   public static final int NUMERIC_TYPE = UAX29URLEmailTokenizer.NUM;
 153
 154   /**
 155    * Chars in class \p{Line_Break = Complex_Context} are from South East Asian
 156    * scripts (Thai, Lao, Myanmar, Khmer, etc.).  Sequences of these are kept
 157    * together as as a single token rather than broken up, because the logic
 158    * required to break them at word boundaries is too complex for UAX#29.
 159    * <p>
 160    * See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA
 161    */
 162   public static final int SOUTH_EAST_ASIAN_TYPE = UAX29URLEmailTokenizer.SOUTHEAST_ASIAN;
 163
 164   public static final int IDEOGRAPHIC_TYPE = UAX29URLEmailTokenizer.IDEOGRAPHIC;
 165
 166   public static final int HIRAGANA_TYPE = UAX29URLEmailTokenizer.HIRAGANA;
 167
 168   public static final int KATAKANA_TYPE = UAX29URLEmailTokenizer.KATAKANA;
 169
 170   public static final int HANGUL_TYPE = UAX29URLEmailTokenizer.HANGUL;
 171
 172   public static final int EMAIL_TYPE = UAX29URLEmailTokenizer.EMAIL;
 173
 174   public static final int URL_TYPE = UAX29URLEmailTokenizer.URL;
 175
 176   public final int yychar()
 177   {
 178     return yychar;
 179   }
 180
 181   /**
 182    * Fills CharTermAttribute with the current token text.
 183    */
 184   public final void getText(CharTermAttribute t) {
 185     t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
 186   }
 187 %}
 188
 189 %%
 190
 191 // UAX#29 WB1.  sot     ÷
 192 //        WB2.          ÷      eot
 193 //
 194 <<EOF>> { return StandardTokenizerInterface.YYEOF; }
 195
 196 {URL}   { return URL_TYPE; }
 197 {EMAIL} { return EMAIL_TYPE; }
 198
 199 // UAX#29 WB8.   Numeric × Numeric
 200 //        WB11.  Numeric (MidNum | MidNumLet) × Numeric
 201 //        WB12.  Numeric × (MidNum | MidNumLet) Numeric
 202 //        WB13a. (ALetter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet
 203 //        WB13b. ExtendNumLet × (ALetter | Numeric | Katakana)
 204 //
 205 {ExtendNumLetEx}* {NumericEx} ({ExtendNumLetEx}+ {NumericEx}
 206                               | {MidNumericEx} {NumericEx}
 207                               | {NumericEx})*
 208 {ExtendNumLetEx}*
 209   { return NUMERIC_TYPE; }
 210
 211 // subset of the below for typing purposes only!
 212 {HangulEx}+
 213   { return HANGUL_TYPE; }
 214
 215 {KatakanaEx}+
 216   { return KATAKANA_TYPE; }
 217
 218 // UAX#29 WB5.   ALetter × ALetter
 219 //        WB6.   ALetter × (MidLetter | MidNumLet) ALetter
 220 //        WB7.   ALetter (MidLetter | MidNumLet) × ALetter
 221 //        WB9.   ALetter × Numeric
 222 //        WB10.  Numeric × ALetter
 223 //        WB13.  Katakana × Katakana
 224 //        WB13a. (ALetter | Numeric | Katakana | ExtendNumLet) × ExtendNumLet
 225 //        WB13b. ExtendNumLet × (ALetter | Numeric | Katakana)
 226 //
 227 {ExtendNumLetEx}*  ( {KatakanaEx} ({ExtendNumLetEx}* {KatakanaEx})*
 228                    | ( {NumericEx}  ({ExtendNumLetEx}+ {NumericEx} | {MidNumericEx} {NumericEx} | {NumericEx})*
 229                      | {ALetterEx}  ({ExtendNumLetEx}+ {ALetterEx} | {MidLetterEx}  {ALetterEx} | {ALetterEx})* )+ )
 230 ({ExtendNumLetEx}+ ( {KatakanaEx} ({ExtendNumLetEx}* {KatakanaEx})*
 231                    | ( {NumericEx}  ({ExtendNumLetEx}+ {NumericEx} | {MidNumericEx} {NumericEx} | {NumericEx})*
 232                      | {ALetterEx}  ({ExtendNumLetEx}+ {ALetterEx} | {MidLetterEx}  {ALetterEx} | {ALetterEx})* )+ ) )*
 233 {ExtendNumLetEx}*
 234   { return WORD_TYPE; }
 235
 236
 237 // From UAX #29:
 238 //
 239 //    [C]haracters with the Line_Break property values of Contingent_Break (CB),
 240 //    Complex_Context (SA/South East Asian), and XX (Unknown) are assigned word
 241 //    boundary property values based on criteria outside of the scope of this
 242 //    annex.  That means that satisfactory treatment of languages like Chinese
 243 //    or Thai requires special handling.
 244 //
 245 // In Unicode 6.0, only one character has the \p{Line_Break = Contingent_Break}
 246 // property: U+FFFC (  ) OBJECT REPLACEMENT CHARACTER.
 247 //
 248 // In the ICU implementation of UAX#29, \p{Line_Break = Complex_Context}
 249 // character sequences (from South East Asian scripts like Thai, Myanmar, Khmer,
 250 // Lao, etc.) are kept together.  This grammar does the same below.
 251 //
 252 // See also the Unicode Line Breaking Algorithm:
 253 //
 254 //    http://www.unicode.org/reports/tr14/#SA
 255 //
 256 {ComplexContext}+ { return SOUTH_EAST_ASIAN_TYPE; }
 257
 258 // UAX#29 WB14.  Any ÷ Any
 259 //
 260 {Han} { return IDEOGRAPHIC_TYPE; }
 261 {Hiragana} { return HIRAGANA_TYPE; }
 262
 263
 264 // UAX#29 WB3.   CR × LF
 265 //        WB3a.  (Newline | CR | LF) ÷
 266 //        WB3b.  ÷ (Newline | CR | LF)
 267 //        WB14.  Any ÷ Any
 268 //
 269 [^] { /* Not numeric, word, ideographic, hiragana, or SE Asian -- ignore it. */ }