1 package org.apache.lucene.analysis.icu;
4 * Licensed to the Apache Software Foundation (ASF) under one or more
5 * contributor license agreements. See the NOTICE file distributed with
6 * this work for additional information regarding copyright ownership.
7 * The ASF licenses this file to You under the Apache License, Version 2.0
8 * (the "License"); you may not use this file except in compliance with
9 * the License. You may obtain a copy of the License at
11 * http://www.apache.org/licenses/LICENSE-2.0
13 * Unless required by applicable law or agreed to in writing, software
14 * distributed under the License is distributed on an "AS IS" BASIS,
15 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 * See the License for the specific language governing permissions and
17 * limitations under the License.
20 import org.apache.lucene.analysis.TokenStream;
22 import com.ibm.icu.text.Normalizer2;
25 * A TokenFilter that applies search term folding to Unicode text,
26 * applying foldings from UTR#30 Character Foldings.
28 * This filter applies the following foldings from the report to unicode text:
32 * <li>Canonical duplicates folding
34 * <li>Diacritic removal (including stroke, hook, descender)
35 * <li>Greek letterforms folding
36 * <li>Han Radical folding
37 * <li>Hebrew Alternates folding
39 * <li>Letterforms folding
40 * <li>Math symbol folding
41 * <li>Multigraph Expansions: All
42 * <li>Native digit folding
43 * <li>No-break folding
44 * <li>Overline folding
45 * <li>Positional forms folding
46 * <li>Small forms folding
48 * <li>Spacing Accents folding
49 * <li>Subscript folding
50 * <li>Superscript folding
51 * <li>Suzhou Numeral folding
53 * <li>Underline folding
54 * <li>Vertical forms folding
58 * Additionally, Default Ignorables are removed, and text is normalized to NFKC.
59 * All foldings, case folding, and normalization mappings are applied recursively
60 * to ensure a fully folded and normalized result.
63 public final class ICUFoldingFilter extends ICUNormalizer2Filter {
64 private static final Normalizer2 normalizer = Normalizer2.getInstance(
65 ICUFoldingFilter.class.getResourceAsStream("utr30.nrm"),
66 "utr30", Normalizer2.Mode.COMPOSE);
69 * Create a new ICUFoldingFilter on the specified input
71 public ICUFoldingFilter(TokenStream input) {
72 super(input, normalizer);