test/test_PythonDirectory.py

   1 # ====================================================================
   2 #   Licensed under the Apache License, Version 2.0 (the "License");
   3 #   you may not use this file except in compliance with the License.
   4 #   You may obtain a copy of the License at
   5 #
   6 #       http://www.apache.org/licenses/LICENSE-2.0
   7 #
   8 #   Unless required by applicable law or agreed to in writing, software
   9 #   distributed under the License is distributed on an "AS IS" BASIS,
  10 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  11 #   See the License for the specific language governing permissions and
  12 #   limitations under the License.
  13 # ====================================================================
  14
  15 import os, sys, unittest, shutil
  16 from threading import RLock
  17 import test_PyLucene
  18
  19 from lucene import \
  20     PythonLock, PythonLockFactory, \
  21     PythonIndexInput, PythonIndexOutput, PythonDirectory, \
  22     JavaError, IOException, JArray, String
  23
  24 """
  25 The Directory Implementation here is for testing purposes only, not meant
  26 as an example of writing one, the implementation here suffers from a lack
  27 of safety when dealing with concurrent modifications as it does away with
  28 the file locking in the default lucene fsdirectory implementation.
  29 """
  30
  31 DEBUG = False
  32
  33 class DebugWrapper(object):
  34
  35     def __init__(self, obj):
  36         self.obj = obj
  37
  38     def __getattr__(self, name):
  39         print self.obj.__class__.__name__, self.obj.name, name
  40         sys.stdout.flush()
  41         return getattr(self.obj, name)
  42
  43 class DebugFactory(object):
  44
  45     def __init__(self, klass):
  46         self.klass = klass
  47
  48     def __call__(self, *args, **kw):
  49         instance = self.klass(*args, **kw)
  50         return DebugWrapper(instance)
  51
  52
  53 class PythonDirLock(PythonLock):
  54     # only safe for a single process
  55
  56     def __init__(self, name, path, lock):
  57         super(PythonDirLock, self).__init__()
  58
  59         self.name = name
  60         self.lock_file = path
  61         self.lock = lock
  62
  63     def isLocked(self):
  64         return self.lock.locked()
  65
  66     def obtain(self):
  67         return self.lock.acquire()
  68
  69     def release(self):
  70         return self.lock.release()
  71
  72
  73 class PythonDirLockFactory(PythonLockFactory):
  74
  75     def __init__(self, path):
  76         super(PythonDirLockFactory, self).__init__()
  77
  78         self.path = path
  79         self._locks = {}
  80
  81     def makeLock(self, name):
  82
  83         lock = self._locks.get(name)
  84         if lock is None:
  85             lock = PythonDirLock(name, os.path.join(self.path, name), RLock())
  86             self._locks[name] = lock
  87
  88         return lock
  89
  90     def clearLock(self, name):
  91
  92         lock = self._locks.pop(name, None)
  93         if lock is not None:
  94             lock.release()
  95
  96
  97 class PythonFileStreamInput(PythonIndexInput):
  98
  99     def __init__(self, name, fh, size, clone=False):
 100         if not clone:
 101             super(PythonFileStreamInput, self).__init__()
 102         self.name = name
 103         self.fh = fh
 104         self._length = size
 105         self.isOpen = True
 106         self.isClone = clone
 107
 108     def length(self):
 109         return long(self._length)
 110
 111     def clone(self):
 112         clone = PythonFileStreamInput(self.name, self.fh, self._length, True)
 113         return super(PythonFileStreamInput, self).clone(clone)
 114
 115     def close(self):
 116         if self.isOpen:
 117             self.isOpen = False
 118             if not self.isClone:
 119                 self.fh.close()
 120
 121     def readInternal(self, length, pos):
 122         self.fh.seek(pos)
 123         return JArray('byte')(self.fh.read(length))
 124
 125     def seekInternal(self, pos):
 126         self.fh.seek(pos)
 127
 128
 129 class PythonFileStreamOutput(PythonIndexOutput):
 130
 131     def __init__(self, name, fh):
 132         super(PythonFileStreamOutput, self).__init__()
 133         self.name = name
 134         self.fh = fh
 135         self.isOpen = True
 136         self._length = 0
 137
 138     def close(self):
 139         if self.isOpen:
 140             super(PythonFileStreamOutput, self).close()
 141             self.isOpen = False
 142             self.fh.close()
 143
 144     def length(self):
 145         return long(self._length)
 146
 147     def seekInternal(self, pos):
 148         self.fh.seek(pos)
 149
 150     def flushBuffer(self, bytes):
 151
 152         self.fh.write(bytes.string_)
 153         self.fh.flush()
 154         self._length += len(bytes)
 155
 156
 157 class PythonFileDirectory(PythonDirectory):
 158
 159     def __init__(self, path):
 160         super(PythonFileDirectory, self).__init__(PythonDirLockFactory(path))
 161
 162         self.name = path
 163         assert os.path.isdir(path)
 164         self.path = path
 165         self._streams = []
 166
 167     def close(self):
 168         for stream in self._streams:
 169             stream.close()
 170         del self._streams[:]
 171
 172     def createOutput(self, name):
 173         file_path = os.path.join(self.path, name)
 174         fh = open(file_path, "wb")
 175         stream = PythonFileStreamOutput(name, fh)
 176         self._streams.append(stream)
 177         return stream
 178
 179     def deleteFile(self, name):
 180         if self.fileExists(name):
 181             os.unlink(os.path.join(self.path, name))
 182
 183     def fileExists(self, name):
 184         return os.path.exists(os.path.join(self.path, name))
 185
 186     def fileLength(self, name):
 187         file_path = os.path.join(self.path, name)
 188         return long(os.path.getsize(file_path))
 189
 190     def fileModified(self, name):
 191         file_path = os.path.join(self.path, name)
 192         return os.path.getmtime(file_path)
 193
 194     def listAll(self):
 195         return os.listdir(self.path)
 196
 197     def sync(self, name):
 198         pass
 199
 200     def openInput(self, name, bufferSize=0):
 201         file_path = os.path.join(self.path, name)
 202         try:
 203             fh = open(file_path, "rb")
 204         except IOError:
 205             raise JavaError, IOException(name)
 206         stream = PythonFileStreamInput(name, fh, os.path.getsize(file_path))
 207         self._streams.append(stream)
 208         return stream
 209
 210     def touchFile(self, name):
 211         file_path = os.path.join(self.path, name)
 212         os.utime(file_path, None)
 213
 214
 215 if DEBUG:
 216     _globals = globals()
 217     _globals['PythonFileDirectory'] = DebugFactory(PythonFileDirectory)
 218     _globals['PythonFileStreamInput'] = DebugFactory(PythonFileStreamInput)
 219     _globals['PythonFileStreamOutput'] = DebugFactory(PythonFileStreamOutput)
 220     _globals['PythonDirLock'] = DebugFactory(PythonDirLock)
 221     del _globals
 222
 223 class PythonDirectoryTests(unittest.TestCase, test_PyLucene.Test_PyLuceneBase):
 224
 225     STORE_DIR = "testpyrepo"
 226
 227     def setUp(self):
 228         if not os.path.exists(self.STORE_DIR):
 229             os.mkdir(self.STORE_DIR)
 230
 231     def tearDown(self):
 232         if os.path.exists(self.STORE_DIR):
 233             shutil.rmtree(self.STORE_DIR)
 234
 235     def openStore(self):
 236         return PythonFileDirectory(self.STORE_DIR)
 237
 238     def closeStore(self, store, *args):
 239         for arg in args:
 240             if arg is not None:
 241                 arg.close()
 242         store.close()
 243
 244     def test_IncrementalLoop(self):
 245         print "Testing Indexing Incremental Looping"
 246         for i in range(100):
 247             print "indexing ", i
 248             sys.stdout.flush()
 249             self.test_indexDocument()
 250
 251
 252 if __name__ == "__main__":
 253     import sys, lucene
 254     env = lucene.initVM()
 255     if '-loop' in sys.argv:
 256         sys.argv.remove('-loop')
 257         while True:
 258             try:
 259                 unittest.main()
 260             except:
 261                 pass
 262             print 'inputs', env._dumpRefs(True).get('class org.osafoundation.lucene.store.PythonIndexOutput', 0)
 263             print 'outputs', env._dumpRefs(True).get('class org.osafoundation.lucene.store.PythonIndexInput', 0)
 264             print 'locks', env._dumpRefs(True).get('class org.osafoundation.lucene.store.PythonLock', 0)
 265             print 'dirs', env._dumpRefs(True).get('class org.osafoundation.lucene.store.PythonLock', 0)
 266     else:
 267         unittest.main()