+++ /dev/null
-/* Generated By:JavaCC: Do not edit this line. HTMLParser.java */
-package org.apache.lucene.benchmark.byTask.feeds.demohtml;
-
-import java.io.*;
-import java.util.Locale;
-import java.util.Properties;
-
-public class HTMLParser implements HTMLParserConstants {
- public static int SUMMARY_LENGTH = 200;
-
- StringBuffer title = new StringBuffer(SUMMARY_LENGTH);
- StringBuffer summary = new StringBuffer(SUMMARY_LENGTH * 2);
- Properties metaTags=new Properties();
- String currentMetaTag=null;
- String currentMetaContent=null;
- int length = 0;
- boolean titleComplete = false;
- boolean inTitle = false;
- boolean inMetaTag = false;
- boolean inStyle = false;
- boolean afterTag = false;
- boolean afterSpace = false;
- String eol = System.getProperty("line.separator");
- Reader pipeIn = null;
- Writer pipeOut;
- private MyPipedInputStream pipeInStream = null;
- private PipedOutputStream pipeOutStream = null;
-
- private class MyPipedInputStream extends PipedInputStream{
-
- public MyPipedInputStream(){
- super();
- }
-
- public MyPipedInputStream(PipedOutputStream src) throws IOException{
- super(src);
- }
-
- public boolean full() throws IOException{
- return this.available() >= PipedInputStream.PIPE_SIZE;
- }
- }
-
- public String getTitle() throws IOException, InterruptedException {
- if (pipeIn == null)
- getReader(); // spawn parsing thread
- while (true) {
- synchronized(this) {
- if (titleComplete || pipeInStream.full())
- break;
- wait(10);
- }
- }
- return title.toString().trim();
- }
-
- public Properties getMetaTags() throws IOException,
-InterruptedException {
- if (pipeIn == null)
- getReader(); // spawn parsing thread
- while (true) {
- synchronized(this) {
- if (titleComplete || pipeInStream.full())
- break;
- wait(10);
- }
- }
- return metaTags;
- }
-
-
- public String getSummary() throws IOException, InterruptedException {
- if (pipeIn == null)
- getReader(); // spawn parsing thread
- while (true) {
- synchronized(this) {
- if (summary.length() >= SUMMARY_LENGTH || pipeInStream.full())
- break;
- wait(10);
- }
- }
- if (summary.length() > SUMMARY_LENGTH)
- summary.setLength(SUMMARY_LENGTH);
-
- String sum = summary.toString().trim();
- String tit = getTitle();
- if (sum.equals(""))
- return tit;
- else
- return sum;
- }
-
- public Reader getReader() throws IOException {
- if (pipeIn == null) {
- pipeInStream = new MyPipedInputStream();
- pipeOutStream = new PipedOutputStream(pipeInStream);
- pipeIn = new InputStreamReader(pipeInStream, "UTF-16BE");
- pipeOut = new OutputStreamWriter(pipeOutStream, "UTF-16BE");
-
- Thread thread = new ParserThread(this);
- thread.start(); // start parsing
- }
-
- return pipeIn;
- }
-
- void addToSummary(String text) {
- if (summary.length() < SUMMARY_LENGTH) {
- summary.append(text);
- if (summary.length() >= SUMMARY_LENGTH) {
- synchronized(this) {
- notifyAll();
- }
- }
- }
- }
-
- void addText(String text) throws IOException {
- if (inStyle)
- return;
- if (inTitle)
- title.append(text);
- else {
- addToSummary(text);
- if (!titleComplete && !(title.length() == 0)) { // finished title
- synchronized(this) {
- titleComplete = true; // tell waiting threads
- notifyAll();
- }
- }
- }
-
- length += text.length();
- pipeOut.write(text);
-
- afterSpace = false;
- }
-
- void addMetaTag() {
- metaTags.setProperty(currentMetaTag, currentMetaContent);
- currentMetaTag = null;
- currentMetaContent = null;
- return;
- }
-
- void addSpace() throws IOException {
- if (!afterSpace) {
- if (inTitle)
- title.append(" ");
- else
- addToSummary(" ");
-
- String space = afterTag ? eol : " ";
- length += space.length();
- pipeOut.write(space);
- afterSpace = true;
- }
- }
-
- final public void HTMLDocument() throws ParseException, IOException {
- Token t;
- label_1:
- while (true) {
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ScriptStart:
- case TagName:
- case DeclName:
- case Comment1:
- case Comment2:
- case Word:
- case Entity:
- case Space:
- case Punct:
- ;
- break;
- default:
- jj_la1[0] = jj_gen;
- break label_1;
- }
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case TagName:
- Tag();
- afterTag = true;
- break;
- case DeclName:
- t = Decl();
- afterTag = true;
- break;
- case Comment1:
- case Comment2:
- CommentTag();
- afterTag = true;
- break;
- case ScriptStart:
- ScriptTag();
- afterTag = true;
- break;
- case Word:
- t = jj_consume_token(Word);
- addText(t.image); afterTag = false;
- break;
- case Entity:
- t = jj_consume_token(Entity);
- addText(Entities.decode(t.image)); afterTag = false;
- break;
- case Punct:
- t = jj_consume_token(Punct);
- addText(t.image); afterTag = false;
- break;
- case Space:
- jj_consume_token(Space);
- addSpace(); afterTag = false;
- break;
- default:
- jj_la1[1] = jj_gen;
- jj_consume_token(-1);
- throw new ParseException();
- }
- }
- jj_consume_token(0);
- }
-
- final public void Tag() throws ParseException, IOException {
- Token t1, t2;
- boolean inImg = false;
- t1 = jj_consume_token(TagName);
- String tagName = t1.image.toLowerCase(Locale.ENGLISH);
- if(Tags.WS_ELEMS.contains(tagName) ) {
- addSpace();
- }
- inTitle = tagName.equalsIgnoreCase("<title"); // keep track if in <TITLE>
- inMetaTag = tagName.equalsIgnoreCase("<META"); // keep track if in <META>
- inStyle = tagName.equalsIgnoreCase("<STYLE"); // keep track if in <STYLE>
- inImg = tagName.equalsIgnoreCase("<img"); // keep track if in <IMG>
-
- label_2:
- while (true) {
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ArgName:
- ;
- break;
- default:
- jj_la1[2] = jj_gen;
- break label_2;
- }
- t1 = jj_consume_token(ArgName);
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ArgEquals:
- jj_consume_token(ArgEquals);
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ArgValue:
- case ArgQuote1:
- case ArgQuote2:
- t2 = ArgValue();
- if (inImg && t1.image.equalsIgnoreCase("alt") && t2 != null)
- addText("[" + t2.image + "]");
-
- if(inMetaTag &&
- ( t1.image.equalsIgnoreCase("name") ||
- t1.image.equalsIgnoreCase("HTTP-EQUIV")
- )
- && t2 != null)
- {
- currentMetaTag=t2.image.toLowerCase(Locale.ENGLISH);
- if(currentMetaTag != null && currentMetaContent != null) {
- addMetaTag();
- }
- }
- if(inMetaTag && t1.image.equalsIgnoreCase("content") && t2 !=
-null)
- {
- currentMetaContent=t2.image.toLowerCase(Locale.ENGLISH);
- if(currentMetaTag != null && currentMetaContent != null) {
- addMetaTag();
- }
- }
- break;
- default:
- jj_la1[3] = jj_gen;
- ;
- }
- break;
- default:
- jj_la1[4] = jj_gen;
- ;
- }
- }
- jj_consume_token(TagEnd);
- }
-
- final public Token ArgValue() throws ParseException {
- Token t = null;
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ArgValue:
- t = jj_consume_token(ArgValue);
- {if (true) return t;}
- break;
- default:
- jj_la1[5] = jj_gen;
- if (jj_2_1(2)) {
- jj_consume_token(ArgQuote1);
- jj_consume_token(CloseQuote1);
- {if (true) return t;}
- } else {
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ArgQuote1:
- jj_consume_token(ArgQuote1);
- t = jj_consume_token(Quote1Text);
- jj_consume_token(CloseQuote1);
- {if (true) return t;}
- break;
- default:
- jj_la1[6] = jj_gen;
- if (jj_2_2(2)) {
- jj_consume_token(ArgQuote2);
- jj_consume_token(CloseQuote2);
- {if (true) return t;}
- } else {
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ArgQuote2:
- jj_consume_token(ArgQuote2);
- t = jj_consume_token(Quote2Text);
- jj_consume_token(CloseQuote2);
- {if (true) return t;}
- break;
- default:
- jj_la1[7] = jj_gen;
- jj_consume_token(-1);
- throw new ParseException();
- }
- }
- }
- }
- }
- throw new Error("Missing return statement in function");
- }
-
- final public Token Decl() throws ParseException {
- Token t;
- t = jj_consume_token(DeclName);
- label_3:
- while (true) {
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ArgName:
- case ArgEquals:
- case ArgValue:
- case ArgQuote1:
- case ArgQuote2:
- ;
- break;
- default:
- jj_la1[8] = jj_gen;
- break label_3;
- }
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ArgName:
- jj_consume_token(ArgName);
- break;
- case ArgValue:
- case ArgQuote1:
- case ArgQuote2:
- ArgValue();
- break;
- case ArgEquals:
- jj_consume_token(ArgEquals);
- break;
- default:
- jj_la1[9] = jj_gen;
- jj_consume_token(-1);
- throw new ParseException();
- }
- }
- jj_consume_token(TagEnd);
- {if (true) return t;}
- throw new Error("Missing return statement in function");
- }
-
- final public void CommentTag() throws ParseException {
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case Comment1:
- jj_consume_token(Comment1);
- label_4:
- while (true) {
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case CommentText1:
- ;
- break;
- default:
- jj_la1[10] = jj_gen;
- break label_4;
- }
- jj_consume_token(CommentText1);
- }
- jj_consume_token(CommentEnd1);
- break;
- case Comment2:
- jj_consume_token(Comment2);
- label_5:
- while (true) {
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case CommentText2:
- ;
- break;
- default:
- jj_la1[11] = jj_gen;
- break label_5;
- }
- jj_consume_token(CommentText2);
- }
- jj_consume_token(CommentEnd2);
- break;
- default:
- jj_la1[12] = jj_gen;
- jj_consume_token(-1);
- throw new ParseException();
- }
- }
-
- final public void ScriptTag() throws ParseException {
- jj_consume_token(ScriptStart);
- label_6:
- while (true) {
- switch ((jj_ntk==-1)?jj_ntk():jj_ntk) {
- case ScriptText:
- ;
- break;
- default:
- jj_la1[13] = jj_gen;
- break label_6;
- }
- jj_consume_token(ScriptText);
- }
- jj_consume_token(ScriptEnd);
- }
-
- private boolean jj_2_1(int xla) {
- jj_la = xla; jj_lastpos = jj_scanpos = token;
- try { return !jj_3_1(); }
- catch(LookaheadSuccess ls) { return true; }
- finally { jj_save(0, xla); }
- }
-
- private boolean jj_2_2(int xla) {
- jj_la = xla; jj_lastpos = jj_scanpos = token;
- try { return !jj_3_2(); }
- catch(LookaheadSuccess ls) { return true; }
- finally { jj_save(1, xla); }
- }
-
- private boolean jj_3_2() {
- if (jj_scan_token(ArgQuote2)) return true;
- if (jj_scan_token(CloseQuote2)) return true;
- return false;
- }
-
- private boolean jj_3_1() {
- if (jj_scan_token(ArgQuote1)) return true;
- if (jj_scan_token(CloseQuote1)) return true;
- return false;
- }
-
- /** Generated Token Manager. */
- public HTMLParserTokenManager token_source;
- SimpleCharStream jj_input_stream;
- /** Current token. */
- public Token token;
- /** Next token. */
- public Token jj_nt;
- private int jj_ntk;
- private Token jj_scanpos, jj_lastpos;
- private int jj_la;
- private int jj_gen;
- final private int[] jj_la1 = new int[14];
- static private int[] jj_la1_0;
- static {
- jj_la1_init_0();
- }
- private static void jj_la1_init_0() {
- jj_la1_0 = new int[] {0x2c7e,0x2c7e,0x10000,0x380000,0x20000,0x80000,0x100000,0x200000,0x3b0000,0x3b0000,0x8000000,0x20000000,0x30,0x4000,};
- }
- final private JJCalls[] jj_2_rtns = new JJCalls[2];
- private boolean jj_rescan = false;
- private int jj_gc = 0;
-
- /** Constructor with InputStream. */
- public HTMLParser(java.io.InputStream stream) {
- this(stream, null);
- }
- /** Constructor with InputStream and supplied encoding */
- public HTMLParser(java.io.InputStream stream, String encoding) {
- try { jj_input_stream = new SimpleCharStream(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
- token_source = new HTMLParserTokenManager(jj_input_stream);
- token = new Token();
- jj_ntk = -1;
- jj_gen = 0;
- for (int i = 0; i < 14; i++) jj_la1[i] = -1;
- for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
- }
-
- /** Reinitialise. */
- public void ReInit(java.io.InputStream stream) {
- ReInit(stream, null);
- }
- /** Reinitialise. */
- public void ReInit(java.io.InputStream stream, String encoding) {
- try { jj_input_stream.ReInit(stream, encoding, 1, 1); } catch(java.io.UnsupportedEncodingException e) { throw new RuntimeException(e); }
- token_source.ReInit(jj_input_stream);
- token = new Token();
- jj_ntk = -1;
- jj_gen = 0;
- for (int i = 0; i < 14; i++) jj_la1[i] = -1;
- for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
- }
-
- /** Constructor. */
- public HTMLParser(java.io.Reader stream) {
- jj_input_stream = new SimpleCharStream(stream, 1, 1);
- token_source = new HTMLParserTokenManager(jj_input_stream);
- token = new Token();
- jj_ntk = -1;
- jj_gen = 0;
- for (int i = 0; i < 14; i++) jj_la1[i] = -1;
- for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
- }
-
- /** Reinitialise. */
- public void ReInit(java.io.Reader stream) {
- jj_input_stream.ReInit(stream, 1, 1);
- token_source.ReInit(jj_input_stream);
- token = new Token();
- jj_ntk = -1;
- jj_gen = 0;
- for (int i = 0; i < 14; i++) jj_la1[i] = -1;
- for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
- }
-
- /** Constructor with generated Token Manager. */
- public HTMLParser(HTMLParserTokenManager tm) {
- token_source = tm;
- token = new Token();
- jj_ntk = -1;
- jj_gen = 0;
- for (int i = 0; i < 14; i++) jj_la1[i] = -1;
- for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
- }
-
- /** Reinitialise. */
- public void ReInit(HTMLParserTokenManager tm) {
- token_source = tm;
- token = new Token();
- jj_ntk = -1;
- jj_gen = 0;
- for (int i = 0; i < 14; i++) jj_la1[i] = -1;
- for (int i = 0; i < jj_2_rtns.length; i++) jj_2_rtns[i] = new JJCalls();
- }
-
- private Token jj_consume_token(int kind) throws ParseException {
- Token oldToken;
- if ((oldToken = token).next != null) token = token.next;
- else token = token.next = token_source.getNextToken();
- jj_ntk = -1;
- if (token.kind == kind) {
- jj_gen++;
- if (++jj_gc > 100) {
- jj_gc = 0;
- for (int i = 0; i < jj_2_rtns.length; i++) {
- JJCalls c = jj_2_rtns[i];
- while (c != null) {
- if (c.gen < jj_gen) c.first = null;
- c = c.next;
- }
- }
- }
- return token;
- }
- token = oldToken;
- jj_kind = kind;
- throw generateParseException();
- }
-
- static private final class LookaheadSuccess extends java.lang.Error { }
- final private LookaheadSuccess jj_ls = new LookaheadSuccess();
- private boolean jj_scan_token(int kind) {
- if (jj_scanpos == jj_lastpos) {
- jj_la--;
- if (jj_scanpos.next == null) {
- jj_lastpos = jj_scanpos = jj_scanpos.next = token_source.getNextToken();
- } else {
- jj_lastpos = jj_scanpos = jj_scanpos.next;
- }
- } else {
- jj_scanpos = jj_scanpos.next;
- }
- if (jj_rescan) {
- int i = 0; Token tok = token;
- while (tok != null && tok != jj_scanpos) { i++; tok = tok.next; }
- if (tok != null) jj_add_error_token(kind, i);
- }
- if (jj_scanpos.kind != kind) return true;
- if (jj_la == 0 && jj_scanpos == jj_lastpos) throw jj_ls;
- return false;
- }
-
-
-/** Get the next Token. */
- final public Token getNextToken() {
- if (token.next != null) token = token.next;
- else token = token.next = token_source.getNextToken();
- jj_ntk = -1;
- jj_gen++;
- return token;
- }
-
-/** Get the specific Token. */
- final public Token getToken(int index) {
- Token t = token;
- for (int i = 0; i < index; i++) {
- if (t.next != null) t = t.next;
- else t = t.next = token_source.getNextToken();
- }
- return t;
- }
-
- private int jj_ntk() {
- if ((jj_nt=token.next) == null)
- return (jj_ntk = (token.next=token_source.getNextToken()).kind);
- else
- return (jj_ntk = jj_nt.kind);
- }
-
- private java.util.List<int[]> jj_expentries = new java.util.ArrayList<int[]>();
- private int[] jj_expentry;
- private int jj_kind = -1;
- private int[] jj_lasttokens = new int[100];
- private int jj_endpos;
-
- private void jj_add_error_token(int kind, int pos) {
- if (pos >= 100) return;
- if (pos == jj_endpos + 1) {
- jj_lasttokens[jj_endpos++] = kind;
- } else if (jj_endpos != 0) {
- jj_expentry = new int[jj_endpos];
- for (int i = 0; i < jj_endpos; i++) {
- jj_expentry[i] = jj_lasttokens[i];
- }
- jj_entries_loop: for (java.util.Iterator it = jj_expentries.iterator(); it.hasNext();) {
- int[] oldentry = (int[])(it.next());
- if (oldentry.length == jj_expentry.length) {
- for (int i = 0; i < jj_expentry.length; i++) {
- if (oldentry[i] != jj_expentry[i]) {
- continue jj_entries_loop;
- }
- }
- jj_expentries.add(jj_expentry);
- break jj_entries_loop;
- }
- }
- if (pos != 0) jj_lasttokens[(jj_endpos = pos) - 1] = kind;
- }
- }
-
- /** Generate ParseException. */
- public ParseException generateParseException() {
- jj_expentries.clear();
- boolean[] la1tokens = new boolean[31];
- if (jj_kind >= 0) {
- la1tokens[jj_kind] = true;
- jj_kind = -1;
- }
- for (int i = 0; i < 14; i++) {
- if (jj_la1[i] == jj_gen) {
- for (int j = 0; j < 32; j++) {
- if ((jj_la1_0[i] & (1<<j)) != 0) {
- la1tokens[j] = true;
- }
- }
- }
- }
- for (int i = 0; i < 31; i++) {
- if (la1tokens[i]) {
- jj_expentry = new int[1];
- jj_expentry[0] = i;
- jj_expentries.add(jj_expentry);
- }
- }
- jj_endpos = 0;
- jj_rescan_token();
- jj_add_error_token(0, 0);
- int[][] exptokseq = new int[jj_expentries.size()][];
- for (int i = 0; i < jj_expentries.size(); i++) {
- exptokseq[i] = jj_expentries.get(i);
- }
- return new ParseException(token, exptokseq, tokenImage);
- }
-
- /** Enable tracing. */
- final public void enable_tracing() {
- }
-
- /** Disable tracing. */
- final public void disable_tracing() {
- }
-
- private void jj_rescan_token() {
- jj_rescan = true;
- for (int i = 0; i < 2; i++) {
- try {
- JJCalls p = jj_2_rtns[i];
- do {
- if (p.gen > jj_gen) {
- jj_la = p.arg; jj_lastpos = jj_scanpos = p.first;
- switch (i) {
- case 0: jj_3_1(); break;
- case 1: jj_3_2(); break;
- }
- }
- p = p.next;
- } while (p != null);
- } catch(LookaheadSuccess ls) { }
- }
- jj_rescan = false;
- }
-
- private void jj_save(int index, int xla) {
- JJCalls p = jj_2_rtns[index];
- while (p.gen > jj_gen) {
- if (p.next == null) { p = p.next = new JJCalls(); break; }
- p = p.next;
- }
- p.gen = jj_gen + xla - jj_la; p.first = token; p.arg = xla;
- }
-
- static final class JJCalls {
- int gen;
- Token first;
- int arg;
- JJCalls next;
- }
-
-// void handleException(Exception e) {
-// System.out.println(e.toString()); // print the error message
-// System.out.println("Skipping...");
-// Token t;
-// do {
-// t = getNextToken();
-// } while (t.kind != TagEnd);
-// }
-}
fnp / pylucene.git / blobdiff