+++ /dev/null
-package org.apache.lucene.analysis.standard;
-
-/*
- * Copyright 2001-2005 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-import java.io.Writer;
-import java.net.URL;
-import java.net.URLConnection;
-import java.text.DateFormat;
-import java.util.Date;
-import java.util.Locale;
-import java.util.SortedSet;
-import java.util.TimeZone;
-import java.util.TreeSet;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-/**
- * Generates a file containing JFlex macros to accept valid ASCII TLDs
- * (top level domains), for inclusion in JFlex grammars that can accept
- * domain names.
- * <p/>
- * The IANA Root Zone Database is queried via HTTP from URL cmdline arg #0, the
- * response is parsed, and the results are written out to a file containing
- * a JFlex macro that will accept all valid ASCII-only TLDs, including punycode
- * forms of internationalized TLDs (output file cmdline arg #1).
- */
-public class GenerateJflexTLDMacros {
-
- public static void main(String... args) throws Exception {
- if (args.length != 2 || args[0].equals("--help") || args[0].equals("-help")) {
- System.err.println("Cmd line params:");
- System.err.println("\tjava " + GenerateJflexTLDMacros.class.getName()
- + "<ZoneFileURL> <JFlexOutputFile>");
- System.exit(1);
- }
- new GenerateJflexTLDMacros(args[0], args[1]).execute();
- }
-
- private static final String NL = System.getProperty("line.separator");
-
- private static final String APACHE_LICENSE
- = "/*" + NL
- + " * Copyright 2001-2005 The Apache Software Foundation." + NL
- + " *" + NL
- + " * Licensed under the Apache License, Version 2.0 (the \"License\");" + NL
- + " * you may not use this file except in compliance with the License." + NL
- + " * You may obtain a copy of the License at" + NL
- + " *" + NL
- + " * http://www.apache.org/licenses/LICENSE-2.0" + NL
- + " *" + NL
- + " * Unless required by applicable law or agreed to in writing, software" + NL
- + " * distributed under the License is distributed on an \"AS IS\" BASIS," + NL
- + " * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied." + NL
- + " * See the License for the specific language governing permissions and" + NL
- + " * limitations under the License." + NL
- + " */" + NL + NL;
-
- private static final Pattern TLD_PATTERN_1
- = Pattern.compile("([-A-Za-z0-9]+)\\.\\s+NS\\s+.*");
- private static final Pattern TLD_PATTERN_2
- = Pattern.compile("([-A-Za-z0-9]+)\\.\\s+\\d+\\s+IN\\s+NS\\s+.*");
- private final URL tldFileURL;
- private long tldFileLastModified = -1L;
- private final File outputFile;
-
- public GenerateJflexTLDMacros(String tldFileURL, String outputFile)
- throws Exception {
- this.tldFileURL = new URL(tldFileURL);
- this.outputFile = new File(outputFile);
- }
-
- /**
- * Downloads the IANA Root Zone Database, extracts the ASCII TLDs, then
- * writes a JFlex macro accepting any of them case-insensitively out to
- * the specified output file.
- *
- * @throws IOException if there is a problem either downloading the database
- * or writing out the output file.
- */
- public void execute() throws IOException {
- final SortedSet<String> TLDs = getIANARootZoneDatabase();
- writeOutput(TLDs);
- System.err.println("Wrote " + TLDs.size() + " top level domains to '"
- + outputFile + "'.");
- }
-
- /**
- * Downloads the IANA Root Zone Database.
- * @return downcased sorted set of ASCII TLDs
- * @throws java.io.IOException if there is a problem downloading the database
- */
- private SortedSet<String> getIANARootZoneDatabase() throws IOException {
- final SortedSet<String> TLDs = new TreeSet<String>();
- final URLConnection connection = tldFileURL.openConnection();
- connection.setUseCaches(false);
- connection.addRequestProperty("Cache-Control", "no-cache");
- connection.connect();
- tldFileLastModified = connection.getLastModified();
- BufferedReader reader = new BufferedReader
- (new InputStreamReader(connection.getInputStream(), "US-ASCII"));
- try {
- String line;
- while (null != (line = reader.readLine())) {
- Matcher matcher = TLD_PATTERN_1.matcher(line);
- if (matcher.matches()) {
- TLDs.add(matcher.group(1).toLowerCase(Locale.US));
- } else {
- matcher = TLD_PATTERN_2.matcher(line);
- if (matcher.matches()) {
- TLDs.add(matcher.group(1).toLowerCase(Locale.US));
- }
- }
- }
- } finally {
- reader.close();
- }
- return TLDs;
- }
-
- /**
- * Writes a file containing a JFlex macro that will accept any of the given
- * TLDs case-insensitively.
- *
- * @param ASCIITLDs The downcased sorted set of top level domains to accept
- * @throws IOException if there is an error writing the output file
- */
- private void writeOutput(SortedSet<String> ASCIITLDs) throws IOException {
- final DateFormat dateFormat = DateFormat.getDateTimeInstance
- (DateFormat.FULL, DateFormat.FULL, Locale.US);
- dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
- final Writer writer = new OutputStreamWriter
- (new FileOutputStream(outputFile), "UTF-8");
- try {
- writer.write(APACHE_LICENSE);
- writer.write("// Generated from IANA Root Zone Database <");
- writer.write(tldFileURL.toString());
- writer.write(">");
- writer.write(NL);
- if (tldFileLastModified > 0L) {
- writer.write("// file version from ");
- writer.write(dateFormat.format(tldFileLastModified));
- writer.write(NL);
- }
- writer.write("// generated on ");
- writer.write(dateFormat.format(new Date()));
- writer.write(NL);
- writer.write("// by ");
- writer.write(this.getClass().getName());
- writer.write(NL);
- writer.write(NL);
- writer.write("ASCIITLD = \".\" (");
- writer.write(NL);
- boolean isFirst = true;
- for (String ASCIITLD : ASCIITLDs) {
- writer.write("\t");
- if (isFirst) {
- isFirst = false;
- writer.write(" ");
- } else {
- writer.write("| ");
- }
- writer.write(getCaseInsensitiveRegex(ASCIITLD));
- writer.write(NL);
- }
- writer.write("\t) \".\"? // Accept trailing root (empty) domain");
- writer.write(NL);
- writer.write(NL);
- } finally {
- writer.close();
- }
- }
-
- /**
- * Returns a regex that will accept the given ASCII TLD case-insensitively.
- *
- * @param ASCIITLD The ASCII TLD to generate a regex for
- * @return a regex that will accept the given ASCII TLD case-insensitively
- */
- private String getCaseInsensitiveRegex(String ASCIITLD) {
- StringBuilder builder = new StringBuilder();
- for (int pos = 0 ; pos < ASCIITLD.length() ; ++pos) {
- char ch = ASCIITLD.charAt(pos);
- if (Character.isDigit(ch) || ch == '-') {
- builder.append(ch);
- } else {
- builder.append("[").append(ch).append(Character.toUpperCase(ch)).append("]");
- }
- }
- return builder.toString();
- }
-}
fnp / pylucene.git / blobdiff