Lucene example source code file (GenerateJflexTLDMacros.java)

This example Lucene source code file (GenerateJflexTLDMacros.java) is included in the DevDaily.com "Java Source Code Warehouse" project. The intent of this project is to help you "Learn Java by Example" ^TM.

Java - Lucene tags/keywords

asciitld, bufferedreader, file, generatejflextldmacros, io, ioexception, ioexception, license, license, net, network, nl, nl, regex, sortedset, string, string, text, tlds, util

The Lucene GenerateJflexTLDMacros.java source code

package org.apache.lucene.analysis.standard;

/*
 * Copyright 2001-2005 The Apache Software Foundation.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.net.URL;
import java.net.URLConnection;
import java.text.DateFormat;
import java.util.Date;
import java.util.Locale;
import java.util.SortedSet;
import java.util.TimeZone;
import java.util.TreeSet;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * Generates a file containing JFlex macros to accept valid ASCII TLDs 
 * (top level domains), for inclusion in JFlex grammars that can accept 
 * domain names.
 * <p/> 
 * The IANA Root Zone Database is queried via HTTP from URL cmdline arg #0, the
 * response is parsed, and the results are written out to a file containing 
 * a JFlex macro that will accept all valid ASCII-only TLDs, including punycode 
 * forms of internationalized TLDs (output file cmdline arg #1).
 */
public class GenerateJflexTLDMacros {

  public static void main(String... args) throws Exception {
    if (args.length != 2 || args[0].equals("--help") || args[0].equals("-help")) {
      System.err.println("Cmd line params:");
      System.err.println("\tjava " + GenerateJflexTLDMacros.class.getName() 
                         + "<ZoneFileURL> ");
      System.exit(1);
    }
    new GenerateJflexTLDMacros(args[0], args[1]).execute();
  }
  
  private static final String NL = System.getProperty("line.separator");
  
  private static final String APACHE_LICENSE 
    = "/*" + NL
      + " * Copyright 2001-2005 The Apache Software Foundation." + NL
      + " *" + NL
      + " * Licensed under the Apache License, Version 2.0 (the \"License\");" + NL
      + " * you may not use this file except in compliance with the License." + NL
      + " * You may obtain a copy of the License at" + NL
      + " *" + NL
      + " *      http://www.apache.org/licenses/LICENSE-2.0" + NL
      + " *" + NL
      + " * Unless required by applicable law or agreed to in writing, software" + NL
      + " * distributed under the License is distributed on an \"AS IS\" BASIS," + NL
      + " * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied." + NL
      + " * See the License for the specific language governing permissions and" + NL
      + " * limitations under the License." + NL
      + " */" + NL + NL;
    
  private static final Pattern TLD_PATTERN_1 
    = Pattern.compile("([-A-Za-z0-9]+)\\.\\s+NS\\s+.*");
  private static final Pattern TLD_PATTERN_2
    = Pattern.compile("([-A-Za-z0-9]+)\\.\\s+\\d+\\s+IN\\s+NS\\s+.*");
  private final URL tldFileURL;
  private long tldFileLastModified = -1L;
  private final File outputFile;

  public GenerateJflexTLDMacros(String tldFileURL, String outputFile)
    throws Exception {
    this.tldFileURL = new URL(tldFileURL);
    this.outputFile = new File(outputFile);
  }

  /**
   * Downloads the IANA Root Zone Database, extracts the ASCII TLDs, then
   * writes a JFlex macro accepting any of them case-insensitively out to
   * the specified output file.
   * 
   * @throws IOException if there is a problem either downloading the database
   *  or writing out the output file.
   */
  public void execute() throws IOException {
    final SortedSet<String> TLDs = getIANARootZoneDatabase();
    writeOutput(TLDs);
    System.err.println("Wrote " + TLDs.size() + " top level domains to '" 
                       + outputFile + "'.");
  }
  
  /**
   * Downloads the IANA Root Zone Database.
   * @return downcased sorted set of ASCII TLDs
   * @throws java.io.IOException if there is a problem downloading the database 
   */
  private SortedSet<String> getIANARootZoneDatabase() throws IOException {
    final SortedSet<String> TLDs = new TreeSet();
    final URLConnection connection = tldFileURL.openConnection();
    connection.setUseCaches(false);
    connection.addRequestProperty("Cache-Control", "no-cache");
    connection.connect();
    tldFileLastModified = connection.getLastModified();
    BufferedReader reader = new BufferedReader
      (new InputStreamReader(connection.getInputStream(), "US-ASCII"));
    try {
      String line;
      while (null != (line = reader.readLine())) {
        Matcher matcher = TLD_PATTERN_1.matcher(line);
        if (matcher.matches()) {
          TLDs.add(matcher.group(1).toLowerCase(Locale.US));
        } else {
          matcher = TLD_PATTERN_2.matcher(line);
          if (matcher.matches()) {
            TLDs.add(matcher.group(1).toLowerCase(Locale.US));
          }
        }
      }
    } finally {
      reader.close();
    }
    return TLDs;
  }

  /**
   * Writes a file containing a JFlex macro that will accept any of the given
   * TLDs case-insensitively.
   * 
   * @param ASCIITLDs The downcased sorted set of top level domains to accept
   * @throws IOException if there is an error writing the output file
   */
  private void writeOutput(SortedSet<String> ASCIITLDs) throws IOException {
    final DateFormat dateFormat = DateFormat.getDateTimeInstance
      (DateFormat.FULL, DateFormat.FULL, Locale.US);
    dateFormat.setTimeZone(TimeZone.getTimeZone("UTC"));
    final Writer writer = new OutputStreamWriter
      (new FileOutputStream(outputFile), "UTF-8");
    try {
      writer.write(APACHE_LICENSE);
      writer.write("// Generated from IANA Root Zone Database <");
      writer.write(tldFileURL.toString());
      writer.write(">");
      writer.write(NL);
      if (tldFileLastModified > 0L) {
        writer.write("// file version from ");
        writer.write(dateFormat.format(tldFileLastModified));
        writer.write(NL);
      }
      writer.write("// generated on ");
      writer.write(dateFormat.format(new Date()));
      writer.write(NL);
      writer.write("// by ");
      writer.write(this.getClass().getName());
      writer.write(NL);
      writer.write(NL);
      writer.write("ASCIITLD = \".\" (");
      writer.write(NL);
      boolean isFirst = true;
      for (String ASCIITLD : ASCIITLDs) {
        writer.write("\t");
        if (isFirst) {
          isFirst = false;
          writer.write("  "); 
        } else {
          writer.write("| "); 
        }
        writer.write(getCaseInsensitiveRegex(ASCIITLD));
        writer.write(NL);
      }
      writer.write("\t) \".\"?   // Accept trailing root (empty) domain");
      writer.write(NL);
      writer.write(NL);
    } finally {
      writer.close();
    }
  }

  /**
   * Returns a regex that will accept the given ASCII TLD case-insensitively.
   * 
   * @param ASCIITLD The ASCII TLD to generate a regex for
   * @return a regex that will accept the given ASCII TLD case-insensitively
   */
  private String getCaseInsensitiveRegex(String ASCIITLD) {
    StringBuilder builder = new StringBuilder();
    for (int pos = 0 ; pos < ASCIITLD.length() ; ++pos) {
      char ch = ASCIITLD.charAt(pos);
      if (Character.isDigit(ch) || ch == '-') {
        builder.append(ch);
      } else {
        builder.append("[").append(ch).append(Character.toUpperCase(ch)).append("]");
      }
    }
    return builder.toString();
  }
}