samples/IndexFiles.py

   1 #!/usr/bin/env python
   2
   3 import sys, os, lucene, threading, time
   4 from datetime import datetime
   5
   6 """
   7 This class is loosely based on the Lucene (java implementation) demo class
   8 org.apache.lucene.demo.IndexFiles.  It will take a directory as an argument
   9 and will index all of the files in that directory and downward recursively.
  10 It will index on the file path, the file name and the file contents.  The
  11 resulting Lucene index will be placed in the current directory and called
  12 'index'.
  13 """
  14
  15 class Ticker(object):
  16
  17     def __init__(self):
  18         self.tick = True
  19
  20     def run(self):
  21         while self.tick:
  22             sys.stdout.write('.')
  23             sys.stdout.flush()
  24             time.sleep(1.0)
  25
  26 class IndexFiles(object):
  27     """Usage: python IndexFiles <doc_directory>"""
  28
  29     def __init__(self, root, storeDir, analyzer):
  30
  31         if not os.path.exists(storeDir):
  32             os.mkdir(storeDir)
  33         store = lucene.SimpleFSDirectory(lucene.File(storeDir))
  34         writer = lucene.IndexWriter(store, analyzer, True,
  35                                     lucene.IndexWriter.MaxFieldLength.LIMITED)
  36         writer.setMaxFieldLength(1048576)
  37         self.indexDocs(root, writer)
  38         ticker = Ticker()
  39         print 'optimizing index',
  40         threading.Thread(target=ticker.run).start()
  41         writer.optimize()
  42         writer.close()
  43         ticker.tick = False
  44         print 'done'
  45
  46     def indexDocs(self, root, writer):
  47         for root, dirnames, filenames in os.walk(root):
  48             for filename in filenames:
  49                 if not filename.endswith('.txt'):
  50                     continue
  51                 print "adding", filename
  52                 try:
  53                     path = os.path.join(root, filename)
  54                     file = open(path)
  55                     contents = unicode(file.read(), 'iso-8859-1')
  56                     file.close()
  57                     doc = lucene.Document()
  58                     doc.add(lucene.Field("name", filename,
  59                                          lucene.Field.Store.YES,
  60                                          lucene.Field.Index.NOT_ANALYZED))
  61                     doc.add(lucene.Field("path", path,
  62                                          lucene.Field.Store.YES,
  63                                          lucene.Field.Index.NOT_ANALYZED))
  64                     if len(contents) > 0:
  65                         doc.add(lucene.Field("contents", contents,
  66                                              lucene.Field.Store.NO,
  67                                              lucene.Field.Index.ANALYZED))
  68                     else:
  69                         print "warning: no content in %s" % filename
  70                     writer.addDocument(doc)
  71                 except Exception, e:
  72                     print "Failed in indexDocs:", e
  73
  74 if __name__ == '__main__':
  75     if len(sys.argv) < 2:
  76         print IndexFiles.__doc__
  77         sys.exit(1)
  78     lucene.initVM()
  79     print 'lucene', lucene.VERSION
  80     start = datetime.now()
  81     try:
  82         IndexFiles(sys.argv[1], "index", lucene.StandardAnalyzer(lucene.Version.LUCENE_CURRENT))
  83         end = datetime.now()
  84         print end - start
  85     except Exception, e:
  86         print "Failed: ", e