2 * Licensed to the Apache Software Foundation (ASF) under one or more
3 * contributor license agreements. See the NOTICE file distributed with
4 * this work for additional information regarding copyright ownership.
5 * The ASF licenses this file to You under the Apache License, Version 2.0
6 * (the "License"); you may not use this file except in compliance with
7 * the License. You may obtain a copy of the License at
9 * http://www.apache.org/licenses/LICENSE-2.0
11 * Unless required by applicable law or agreed to in writing, software
12 * distributed under the License is distributed on an "AS IS" BASIS,
13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 * See the License for the specific language governing permissions and
15 * limitations under the License.
18 package org.apache.lucene.analysis.cn.smart;
20 import java.io.IOException;
21 import java.util.Iterator;
22 import java.util.List;
24 import org.apache.lucene.analysis.TokenFilter;
25 import org.apache.lucene.analysis.TokenStream;
26 import org.apache.lucene.analysis.cn.smart.hhmm.SegToken;
27 import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
28 import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
29 import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
32 * A {@link TokenFilter} that breaks sentences into words.
33 * @lucene.experimental
35 public final class WordTokenFilter extends TokenFilter {
37 private WordSegmenter wordSegmenter;
39 private Iterator<SegToken> tokenIter;
41 private List<SegToken> tokenBuffer;
43 private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
44 private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
45 private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
48 * Construct a new WordTokenizer.
50 * @param in {@link TokenStream} of sentences
52 public WordTokenFilter(TokenStream in) {
54 this.wordSegmenter = new WordSegmenter();
58 public boolean incrementToken() throws IOException {
59 if (tokenIter == null || !tokenIter.hasNext()) {
60 // there are no remaining tokens from the current sentence... are there more sentences?
61 if (input.incrementToken()) {
62 // a new sentence is available: process it.
63 tokenBuffer = wordSegmenter.segmentSentence(termAtt.toString(), offsetAtt.startOffset());
64 tokenIter = tokenBuffer.iterator();
66 * it should not be possible to have a sentence with 0 words, check just in case.
67 * returning EOS isn't the best either, but its the behavior of the original code.
69 if (!tokenIter.hasNext())
72 return false; // no more sentences, end of stream!
75 // WordTokenFilter must clear attributes, as it is creating new tokens.
77 // There are remaining tokens from the current sentence, return the next one.
78 SegToken nextWord = tokenIter.next();
79 termAtt.copyBuffer(nextWord.charArray, 0, nextWord.charArray.length);
80 offsetAtt.setOffset(nextWord.startOffset, nextWord.endOffset);
81 typeAtt.setType("word");
86 public void reset() throws IOException {