You must specify the required {@link Version} * compatibility when creating UAX29URLEmailTokenizer: *
* See Unicode Line Breaking Algorithm: http://www.unicode.org/reports/tr14/#SA
* @deprecated use {@link #TOKEN_TYPES} instead
*/
@Deprecated
public static final String SOUTH_EAST_ASIAN_TYPE = TOKEN_TYPES[SOUTHEAST_ASIAN];
/** @deprecated use {@link #TOKEN_TYPES} instead */
@Deprecated
public static final String IDEOGRAPHIC_TYPE = TOKEN_TYPES[IDEOGRAPHIC];
/** @deprecated use {@link #TOKEN_TYPES} instead */
@Deprecated
public static final String HIRAGANA_TYPE = TOKEN_TYPES[HIRAGANA];
/** @deprecated use {@link #TOKEN_TYPES} instead */
@Deprecated
public static final String KATAKANA_TYPE = TOKEN_TYPES[KATAKANA];
/** @deprecated use {@link #TOKEN_TYPES} instead */
@Deprecated
public static final String HANGUL_TYPE = TOKEN_TYPES[HANGUL];
private int maxTokenLength = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;
/** Set the max allowed token length. Any token longer
* than this is skipped. */
public void setMaxTokenLength(int length) {
this.maxTokenLength = length;
}
/** @see #setMaxTokenLength */
public int getMaxTokenLength() {
return maxTokenLength;
}
/** @deprecated use {@link #UAX29URLEmailTokenizer(Version, Reader)} instead. */
@Deprecated
public UAX29URLEmailTokenizer(Reader input) {
this(Version.LUCENE_31, input);
}
/** @deprecated use {@link #UAX29URLEmailTokenizer(Version, Reader)} instead. */
@Deprecated
public UAX29URLEmailTokenizer(InputStream input) {
this(Version.LUCENE_31, new InputStreamReader(input));
}
/** @deprecated use {@link #UAX29URLEmailTokenizer(Version, AttributeSource, Reader)} instead. */
@Deprecated
public UAX29URLEmailTokenizer(AttributeSource source, Reader input) {
this(Version.LUCENE_31, source, input);
}
/** @deprecated use {@link #UAX29URLEmailTokenizer(Version, AttributeSource.AttributeFactory, Reader)} instead. */
@Deprecated
public UAX29URLEmailTokenizer(AttributeFactory factory, Reader input) {
this(Version.LUCENE_31, factory, input);
}
/**
* Creates a new instance of the UAX29URLEmailTokenizer. Attaches
* the input to the newly created JFlex scanner.
*
* @param input The input reader
*/
public UAX29URLEmailTokenizer(Version matchVersion, Reader input) {
super(input);
this.scanner = getScannerFor(matchVersion, input);
}
/**
* Creates a new UAX29URLEmailTokenizer with a given {@link AttributeSource}.
*/
public UAX29URLEmailTokenizer(Version matchVersion, AttributeSource source, Reader input) {
super(source, input);
this.scanner = getScannerFor(matchVersion, input);
}
/**
* Creates a new UAX29URLEmailTokenizer with a given {@link AttributeFactory}
*/
public UAX29URLEmailTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
super(factory, input);
this.scanner = getScannerFor(matchVersion, input);
}
private static StandardTokenizerInterface getScannerFor(Version matchVersion, Reader input) {
if (matchVersion.onOrAfter(Version.LUCENE_34)) {
return new UAX29URLEmailTokenizerImpl(input);
} else {
return new UAX29URLEmailTokenizerImpl31(input);
}
}
// this tokenizer generates three attributes:
// term offset, positionIncrement and type
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
@Override
public final boolean incrementToken() throws IOException {
clearAttributes();
int posIncr = 1;
while(true) {
int tokenType = scanner.getNextToken();
if (tokenType == StandardTokenizerInterface.YYEOF) {
return false;
}
if (scanner.yylength() <= maxTokenLength) {
posIncrAtt.setPositionIncrement(posIncr);
scanner.getText(termAtt);
final int start = scanner.yychar();
offsetAtt.setOffset(correctOffset(start), correctOffset(start+termAtt.length()));
typeAtt.setType(TOKEN_TYPES[tokenType]);
return true;
} else
// When we skip a too-long term, we still increment the
// position increment
posIncr++;
}
}
@Override
public final void end() {
// set final offset
int finalOffset = correctOffset(scanner.yychar() + scanner.yylength());
offsetAtt.setOffset(finalOffset, finalOffset);
}
@Override
public void reset(Reader reader) throws IOException {
super.reset(reader);
scanner.yyreset(reader);
}
}