--- /dev/null
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// HTMLParser.jj
+
+options {
+ STATIC = false;
+ //DEBUG_LOOKAHEAD = true;
+ //DEBUG_TOKEN_MANAGER = true;
+ UNICODE_INPUT = true;
+}
+
+PARSER_BEGIN(HTMLParser)
+
+package org.apache.lucene.benchmark.byTask.feeds.demohtml;
+
+import java.io.*;
+import java.util.Locale;
+import java.util.Properties;
+
+public class HTMLParser {
+ public static int SUMMARY_LENGTH = 200;
+
+ StringBuffer title = new StringBuffer(SUMMARY_LENGTH);
+ StringBuffer summary = new StringBuffer(SUMMARY_LENGTH * 2);
+ Properties metaTags=new Properties();
+ String currentMetaTag=null;
+ String currentMetaContent=null;
+ int length = 0;
+ boolean titleComplete = false;
+ boolean inTitle = false;
+ boolean inMetaTag = false;
+ boolean inStyle = false;
+ boolean afterTag = false;
+ boolean afterSpace = false;
+ String eol = System.getProperty("line.separator");
+ Reader pipeIn = null;
+ Writer pipeOut;
+ private MyPipedInputStream pipeInStream = null;
+ private PipedOutputStream pipeOutStream = null;
+
+ private class MyPipedInputStream extends PipedInputStream{
+
+ public MyPipedInputStream(){
+ super();
+ }
+
+ public MyPipedInputStream(PipedOutputStream src) throws IOException{
+ super(src);
+ }
+
+ public boolean full() throws IOException{
+ return this.available() >= PipedInputStream.PIPE_SIZE;
+ }
+ }
+
+ public String getTitle() throws IOException, InterruptedException {
+ if (pipeIn == null)
+ getReader(); // spawn parsing thread
+ while (true) {
+ synchronized(this) {
+ if (titleComplete || pipeInStream.full())
+ break;
+ wait(10);
+ }
+ }
+ return title.toString().trim();
+ }
+
+ public Properties getMetaTags() throws IOException,
+InterruptedException {
+ if (pipeIn == null)
+ getReader(); // spawn parsing thread
+ while (true) {
+ synchronized(this) {
+ if (titleComplete || pipeInStream.full())
+ break;
+ wait(10);
+ }
+ }
+ return metaTags;
+ }
+
+
+ public String getSummary() throws IOException, InterruptedException {
+ if (pipeIn == null)
+ getReader(); // spawn parsing thread
+ while (true) {
+ synchronized(this) {
+ if (summary.length() >= SUMMARY_LENGTH || pipeInStream.full())
+ break;
+ wait(10);
+ }
+ }
+ if (summary.length() > SUMMARY_LENGTH)
+ summary.setLength(SUMMARY_LENGTH);
+
+ String sum = summary.toString().trim();
+ String tit = getTitle();
+ if (sum.equals(""))
+ return tit;
+ else
+ return sum;
+ }
+
+ public Reader getReader() throws IOException {
+ if (pipeIn == null) {
+ pipeInStream = new MyPipedInputStream();
+ pipeOutStream = new PipedOutputStream(pipeInStream);
+ pipeIn = new InputStreamReader(pipeInStream, "UTF-16BE");
+ pipeOut = new OutputStreamWriter(pipeOutStream, "UTF-16BE");
+
+ Thread thread = new ParserThread(this);
+ thread.start(); // start parsing
+ }
+
+ return pipeIn;
+ }
+
+ void addToSummary(String text) {
+ if (summary.length() < SUMMARY_LENGTH) {
+ summary.append(text);
+ if (summary.length() >= SUMMARY_LENGTH) {
+ synchronized(this) {
+ notifyAll();
+ }
+ }
+ }
+ }
+
+ void addText(String text) throws IOException {
+ if (inStyle)
+ return;
+ if (inTitle)
+ title.append(text);
+ else {
+ addToSummary(text);
+ if (!titleComplete && !(title.length() == 0)) { // finished title
+ synchronized(this) {
+ titleComplete = true; // tell waiting threads
+ notifyAll();
+ }
+ }
+ }
+
+ length += text.length();
+ pipeOut.write(text);
+
+ afterSpace = false;
+ }
+
+ void addMetaTag() {
+ metaTags.setProperty(currentMetaTag, currentMetaContent);
+ currentMetaTag = null;
+ currentMetaContent = null;
+ return;
+ }
+
+ void addSpace() throws IOException {
+ if (!afterSpace) {
+ if (inTitle)
+ title.append(" ");
+ else
+ addToSummary(" ");
+
+ String space = afterTag ? eol : " ";
+ length += space.length();
+ pipeOut.write(space);
+ afterSpace = true;
+ }
+ }
+
+// void handleException(Exception e) {
+// System.out.println(e.toString()); // print the error message
+// System.out.println("Skipping...");
+// Token t;
+// do {
+// t = getNextToken();
+// } while (t.kind != TagEnd);
+// }
+}
+
+PARSER_END(HTMLParser)
+
+
+void HTMLDocument() throws IOException :
+{
+ Token t;
+}
+{
+// try {
+ ( Tag() { afterTag = true; }
+ | t=Decl() { afterTag = true; }
+ | CommentTag() { afterTag = true; }
+ | ScriptTag() { afterTag = true; }
+ | t=<Word> { addText(t.image); afterTag = false; }
+ | t=<Entity> { addText(Entities.decode(t.image)); afterTag = false; }
+ | t=<Punct> { addText(t.image); afterTag = false; }
+ | <Space> { addSpace(); afterTag = false; }
+ )* <EOF>
+// } catch (ParseException e) {
+// handleException(e);
+// }
+}
+
+void Tag() throws IOException :
+{
+ Token t1, t2;
+ boolean inImg = false;
+}
+{
+ t1=<TagName> {
+ String tagName = t1.image.toLowerCase(Locale.ENGLISH);
+ if(Tags.WS_ELEMS.contains(tagName) ) {
+ addSpace();
+ }
+ inTitle = tagName.equalsIgnoreCase("<title"); // keep track if in <TITLE>
+ inMetaTag = tagName.equalsIgnoreCase("<META"); // keep track if in <META>
+ inStyle = tagName.equalsIgnoreCase("<STYLE"); // keep track if in <STYLE>
+ inImg = tagName.equalsIgnoreCase("<img"); // keep track if in <IMG>
+ }
+ (t1=<ArgName>
+ (<ArgEquals>
+ (t2=ArgValue() // save ALT text in IMG tag
+ {
+ if (inImg && t1.image.equalsIgnoreCase("alt") && t2 != null)
+ addText("[" + t2.image + "]");
+
+ if(inMetaTag &&
+ ( t1.image.equalsIgnoreCase("name") ||
+ t1.image.equalsIgnoreCase("HTTP-EQUIV")
+ )
+ && t2 != null)
+ {
+ currentMetaTag=t2.image.toLowerCase(Locale.ENGLISH);
+ if(currentMetaTag != null && currentMetaContent != null) {
+ addMetaTag();
+ }
+ }
+ if(inMetaTag && t1.image.equalsIgnoreCase("content") && t2 !=
+null)
+ {
+ currentMetaContent=t2.image.toLowerCase(Locale.ENGLISH);
+ if(currentMetaTag != null && currentMetaContent != null) {
+ addMetaTag();
+ }
+ }
+ }
+ )?
+ )?
+ )*
+ <TagEnd>
+}
+
+Token ArgValue() :
+{
+ Token t = null;
+}
+{
+ t=<ArgValue> { return t; }
+| LOOKAHEAD(2)
+ <ArgQuote1> <CloseQuote1> { return t; }
+| <ArgQuote1> t=<Quote1Text> <CloseQuote1> { return t; }
+| LOOKAHEAD(2)
+ <ArgQuote2> <CloseQuote2> { return t; }
+| <ArgQuote2> t=<Quote2Text> <CloseQuote2> { return t; }
+}
+
+
+Token Decl() :
+{
+ Token t;
+}
+{
+ t=<DeclName> ( <ArgName> | ArgValue() | <ArgEquals> )* <TagEnd>
+ { return t; }
+}
+
+
+void CommentTag() :
+{}
+{
+ (<Comment1> ( <CommentText1> )* <CommentEnd1>)
+ |
+ (<Comment2> ( <CommentText2> )* <CommentEnd2>)
+}
+
+void ScriptTag() :
+{}
+{
+ <ScriptStart> ( <ScriptText> )* <ScriptEnd>
+}
+
+
+TOKEN :
+{
+ < ScriptStart: "<script" > : WithinScript
+| < TagName: "<" ("/")? ["A"-"Z","a"-"z"] (<ArgName>)? > : WithinTag
+| < DeclName: "<" "!" ["A"-"Z","a"-"z"] (<ArgName>)? > : WithinTag
+
+| < Comment1: "<!--" > : WithinComment1
+| < Comment2: "<!" > : WithinComment2
+
+| < Word: ( <LET> | <LET> (["+","/"])+ | <NUM> ["\""] |
+ <LET> ["-","'"] <LET> | ("$")? <NUM> [",","."] <NUM> )+ >
+| < #LET: ["A"-"Z","a"-"z","0"-"9"] >
+| < #NUM: ["0"-"9"] >
+| < #HEX: ["0"-"9","A"-"F","a"-"f"] >
+
+| < Entity: ( "&" (["A"-"Z","a"-"z"])+ (";")? | "&" "#" (<NUM>)+ (";")? | "&" "#" ["X","x"] (<HEX>)+ (";")? ) >
+
+| < Space: (<SP>)+ >
+| < #SP: [" ","\t","\r","\n"] >
+
+| < Punct: ~[] > // Keep this last. It is a catch-all.
+}
+
+<WithinScript> TOKEN:
+{
+ < ScriptText: (~["<",">"])+ | "<" | ">" >
+| < ScriptEnd: "</script" (~["<",">"])* ">" > : DEFAULT
+}
+
+<WithinTag> TOKEN:
+{
+ < ArgName: (~[" ","\t","\r","\n","=",">","'","\""])
+ (~[" ","\t","\r","\n","=",">"])* >
+| < ArgEquals: "=" > : AfterEquals
+| < TagEnd: ">" | "=>" > : DEFAULT
+}
+
+<AfterEquals> TOKEN:
+{
+ < ArgValue: (~[" ","\t","\r","\n","=",">","'","\""])
+ (~[" ","\t","\r","\n",">"])* > : WithinTag
+}
+
+<WithinTag, AfterEquals> TOKEN:
+{
+ < ArgQuote1: "'" > : WithinQuote1
+| < ArgQuote2: "\"" > : WithinQuote2
+}
+
+<WithinTag, AfterEquals> SKIP:
+{
+ < <Space> >
+}
+
+<WithinQuote1> TOKEN:
+{
+ < Quote1Text: (~["'"])+ >
+| < CloseQuote1: <ArgQuote1> > : WithinTag
+}
+
+<WithinQuote2> TOKEN:
+{
+ < Quote2Text: (~["\""])+ >
+| < CloseQuote2: <ArgQuote2> > : WithinTag
+}
+
+
+<WithinComment1> TOKEN :
+{
+ < CommentText1: (~["-"])+ | "-" >
+| < CommentEnd1: "-->" > : DEFAULT
+}
+
+<WithinComment2> TOKEN :
+{
+ < CommentText2: (~[">"])+ >
+| < CommentEnd2: ">" > : DEFAULT
+}
fnp / pylucene.git / blobdiff