--- /dev/null
+package org.apache.lucene.benchmark.utils;
+
+/**
+ * Copyright 2005 The Apache Software Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Split the Reuters SGML documents into Simple Text files containing: Title,
+ * Date, Dateline, Body
+ */
+public class ExtractReuters {
+ private File reutersDir;
+ private File outputDir;
+ private static final String LINE_SEPARATOR = System.getProperty("line.separator");
+
+ public ExtractReuters(File reutersDir, File outputDir) {
+ this.reutersDir = reutersDir;
+ this.outputDir = outputDir;
+ System.out.println("Deleting all files in " + outputDir);
+ for (File f : outputDir.listFiles()) {
+ f.delete();
+ }
+ }
+
+ public void extract() {
+ File[] sgmFiles = reutersDir.listFiles(new FileFilter() {
+ public boolean accept(File file) {
+ return file.getName().endsWith(".sgm");
+ }
+ });
+ if (sgmFiles != null && sgmFiles.length > 0) {
+ for (File sgmFile : sgmFiles) {
+ extractFile(sgmFile);
+ }
+ } else {
+ System.err.println("No .sgm files in " + reutersDir);
+ }
+ }
+
+ Pattern EXTRACTION_PATTERN = Pattern
+ .compile("<TITLE>(.*?)</TITLE>|<DATE>(.*?)</DATE>|<BODY>(.*?)</BODY>");
+
+ private static String[] META_CHARS = { "&", "<", ">", "\"", "'" };
+
+ private static String[] META_CHARS_SERIALIZATIONS = { "&", "<",
+ ">", """, "'" };
+
+ /**
+ * Override if you wish to change what is extracted
+ *
+ * @param sgmFile
+ */
+ protected void extractFile(File sgmFile) {
+ try {
+ BufferedReader reader = new BufferedReader(new FileReader(sgmFile));
+
+ StringBuilder buffer = new StringBuilder(1024);
+ StringBuilder outBuffer = new StringBuilder(1024);
+
+ String line = null;
+ int docNumber = 0;
+ while ((line = reader.readLine()) != null) {
+ // when we see a closing reuters tag, flush the file
+
+ if (line.indexOf("</REUTERS") == -1) {
+ // Replace the SGM escape sequences
+
+ buffer.append(line).append(' ');// accumulate the strings for now,
+ // then apply regular expression to
+ // get the pieces,
+ } else {
+ // Extract the relevant pieces and write to a file in the output dir
+ Matcher matcher = EXTRACTION_PATTERN.matcher(buffer);
+ while (matcher.find()) {
+ for (int i = 1; i <= matcher.groupCount(); i++) {
+ if (matcher.group(i) != null) {
+ outBuffer.append(matcher.group(i));
+ }
+ }
+ outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
+ }
+ String out = outBuffer.toString();
+ for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) {
+ out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
+ }
+ File outFile = new File(outputDir, sgmFile.getName() + "-"
+ + (docNumber++) + ".txt");
+ // System.out.println("Writing " + outFile);
+ FileWriter writer = new FileWriter(outFile);
+ writer.write(out);
+ writer.close();
+ outBuffer.setLength(0);
+ buffer.setLength(0);
+ }
+ }
+ reader.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static void main(String[] args) {
+ if (args.length != 2) {
+ usage("Wrong number of arguments ("+args.length+")");
+ return;
+ }
+ File reutersDir = new File(args[0]);
+ if (!reutersDir.exists()) {
+ usage("Cannot find Path to Reuters SGM files ("+reutersDir+")");
+ return;
+ }
+
+ // First, extract to a tmp directory and only if everything succeeds, rename
+ // to output directory.
+ File outputDir = new File(args[1]);
+ outputDir = new File(outputDir.getAbsolutePath() + "-tmp");
+ outputDir.mkdirs();
+ ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
+ extractor.extract();
+ // Now rename to requested output dir
+ outputDir.renameTo(new File(args[1]));
+ }
+
+ private static void usage(String msg) {
+ System.err.println("Usage: "+msg+" :: java -cp <...> org.apache.lucene.benchmark.utils.ExtractReuters <Path to Reuters SGM files> <Output Path>");
+ }
+
+}
fnp / pylucene.git / blobdiff