1 # ====================================================================
2 # Licensed under the Apache License, Version 2.0 (the "License");
3 # you may not use this file except in compliance with the License.
4 # You may obtain a copy of the License at
6 # http://www.apache.org/licenses/LICENSE-2.0
8 # Unless required by applicable law or agreed to in writing, software
9 # distributed under the License is distributed on an "AS IS" BASIS,
10 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11 # See the License for the specific language governing permissions and
12 # limitations under the License.
13 # ====================================================================
15 # This sample illustrates how to write an Analyzer 'extension' in Python.
17 # What is happening behind the scenes ?
19 # The PorterStemmerAnalyzer python class does not in fact extend Analyzer,
20 # it merely provides an implementation for Analyzer's abstract tokenStream()
21 # method. When an instance of PorterStemmerAnalyzer is passed to PyLucene,
22 # with a call to IndexWriter(store, PorterStemmerAnalyzer(), True) for
23 # example, the PyLucene SWIG-based glue code wraps it into an instance of
24 # PythonAnalyzer, a proper java extension of Analyzer which implements a
25 # native tokenStream() method whose job is to call the tokenStream() method
26 # on the python instance it wraps. The PythonAnalyzer instance is the
27 # Analyzer extension bridge to PorterStemmerAnalyzer.
30 from datetime import datetime
32 from IndexFiles import IndexFiles
35 class PorterStemmerAnalyzer(PythonAnalyzer):
37 def tokenStream(self, fieldName, reader):
39 result = StandardTokenizer(Version.LUCENE_CURRENT, reader)
40 result = StandardFilter(result)
41 result = LowerCaseFilter(result)
42 result = PorterStemFilter(result)
43 result = StopFilter(True, result, StopAnalyzer.ENGLISH_STOP_WORDS_SET)
48 if __name__ == '__main__':
50 print IndexFiles.__doc__
53 print 'lucene', VERSION
54 start = datetime.now()
56 IndexFiles(sys.argv[1], "index", PorterStemmerAnalyzer())