+++ /dev/null
-/*
- Egothor Software License version 1.00
- Copyright (C) 1997-2004 Leo Galambos.
- Copyright (C) 2002-2004 "Egothor developers"
- on behalf of the Egothor Project.
- All rights reserved.
-
- This software is copyrighted by the "Egothor developers". If this
- license applies to a single file or document, the "Egothor developers"
- are the people or entities mentioned as copyright holders in that file
- or document. If this license applies to the Egothor project as a
- whole, the copyright holders are the people or entities mentioned in
- the file CREDITS. This file can be found in the same location as this
- license in the distribution.
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions are
- met:
- 1. Redistributions of source code must retain the above copyright
- notice, the list of contributors, this list of conditions, and the
- following disclaimer.
- 2. Redistributions in binary form must reproduce the above copyright
- notice, the list of contributors, this list of conditions, and the
- disclaimer that follows these conditions in the documentation
- and/or other materials provided with the distribution.
- 3. The name "Egothor" must not be used to endorse or promote products
- derived from this software without prior written permission. For
- written permission, please contact Leo.G@seznam.cz
- 4. Products derived from this software may not be called "Egothor",
- nor may "Egothor" appear in their name, without prior written
- permission from Leo.G@seznam.cz.
-
- In addition, we request that you include in the end-user documentation
- provided with the redistribution and/or in the software itself an
- acknowledgement equivalent to the following:
- "This product includes software developed by the Egothor Project.
- http://egothor.sf.net/"
-
- THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
- WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
- MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- IN NO EVENT SHALL THE EGOTHOR PROJECT OR ITS CONTRIBUTORS BE LIABLE
- FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
- CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
- SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
- BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
- OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN
- IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
- This software consists of voluntary contributions made by many
- individuals on behalf of the Egothor Project and was originally
- created by Leo Galambos (Leo.G@seznam.cz).
- */
-package org.egothor.stemmer;
-
-import java.io.BufferedOutputStream;
-import java.io.BufferedReader;
-import java.io.DataOutputStream;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.LineNumberReader;
-import java.util.StringTokenizer;
-
-/**
- * The Compile class is used to compile a stemmer table.
- */
-public class Compile {
-
- static boolean backward;
- static boolean multi;
- static Trie trie;
-
- /**
- * Entry point to the Compile application.
- * <p>
- * This program takes any number of arguments: the first is the name of the
- * desired stemming algorithm to use (a list is available in the package
- * description) , all of the rest should be the path or paths to a file or
- * files containing a stemmer table to compile.
- *
- * @param args the command line arguments
- */
- public static void main(java.lang.String[] args) {
- if (args.length < 1) {
- return;
- }
-
- args[0].toUpperCase();
-
- backward = args[0].charAt(0) == '-';
- int qq = (backward) ? 1 : 0;
- boolean storeorig = false;
-
- if (args[0].charAt(qq) == '0') {
- storeorig = true;
- qq++;
- }
-
- multi = args[0].charAt(qq) == 'M';
- if (multi) {
- qq++;
- }
-
- String charset = System.getProperty("egothor.stemmer.charset", "UTF-8");
-
- char optimizer[] = new char[args[0].length() - qq];
- for (int i = 0; i < optimizer.length; i++) {
- optimizer[i] = args[0].charAt(qq + i);
- }
-
- for (int i = 1; i < args.length; i++) {
- LineNumberReader in;
- // System.out.println("[" + args[i] + "]");
- Diff diff = new Diff();
- try {
- int stems = 0;
- int words = 0;
-
- allocTrie();
-
- System.out.println(args[i]);
- in = new LineNumberReader(new BufferedReader(new InputStreamReader(
- new FileInputStream(args[i]), charset)));
- for (String line = in.readLine(); line != null; line = in.readLine()) {
- try {
- line = line.toLowerCase();
- StringTokenizer st = new StringTokenizer(line);
- String stem = st.nextToken();
- if (storeorig) {
- trie.add(stem, "-a");
- words++;
- }
- while (st.hasMoreTokens()) {
- String token = st.nextToken();
- if (token.equals(stem) == false) {
- trie.add(token, diff.exec(token, stem));
- words++;
- }
- }
- } catch (java.util.NoSuchElementException x) {
- // no base token (stem) on a line
- }
- }
-
- Optimizer o = new Optimizer();
- Optimizer2 o2 = new Optimizer2();
- Lift l = new Lift(true);
- Lift e = new Lift(false);
- Gener g = new Gener();
-
- for (int j = 0; j < optimizer.length; j++) {
- String prefix;
- switch (optimizer[j]) {
- case 'G':
- trie = trie.reduce(g);
- prefix = "G: ";
- break;
- case 'L':
- trie = trie.reduce(l);
- prefix = "L: ";
- break;
- case 'E':
- trie = trie.reduce(e);
- prefix = "E: ";
- break;
- case '2':
- trie = trie.reduce(o2);
- prefix = "2: ";
- break;
- case '1':
- trie = trie.reduce(o);
- prefix = "1: ";
- break;
- default:
- continue;
- }
- trie.printInfo(prefix + " ");
- }
-
- DataOutputStream os = new DataOutputStream(new BufferedOutputStream(
- new FileOutputStream(args[i] + ".out")));
- os.writeUTF(args[0]);
- trie.store(os);
- os.close();
-
- } catch (FileNotFoundException x) {
- x.printStackTrace();
- } catch (IOException x) {
- x.printStackTrace();
- }
- }
- }
-
- static void allocTrie() {
- if (multi) {
- trie = new MultiTrie2(!backward);
- } else {
- trie = new Trie(!backward);
- }
- }
-}
fnp / pylucene.git / blobdiff