fnp
/
wolnelektury.git
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
index indictor
[wolnelektury.git]
/
apps
/
search
/
index.py
diff --git
a/apps/search/index.py
b/apps/search/index.py
index
33f2aa1
..
f5489a5
100644
(file)
--- a/
apps/search/index.py
+++ b/
apps/search/index.py
@@
-9,11
+9,13
@@
from lucene import SimpleFSDirectory, IndexWriter, File, Field, \
HashSet, BooleanClause, Term, CharTermAttribute, \
PhraseQuery, StringReader
# KeywordAnalyzer
HashSet, BooleanClause, Term, CharTermAttribute, \
PhraseQuery, StringReader
# KeywordAnalyzer
+import sys
import os
import errno
from librarian import dcparser
from librarian.parser import WLDocument
import catalogue.models
import os
import errno
from librarian import dcparser
from librarian.parser import WLDocument
import catalogue.models
+from multiprocessing.pool import ThreadPool
import atexit
import atexit
@@
-247,24
+249,43
@@
class ReusableIndex(Index):
if you cannot rely on atexit, use ReusableIndex.close_reusable() yourself.
"""
index = None
if you cannot rely on atexit, use ReusableIndex.close_reusable() yourself.
"""
index = None
- def open(self, analyzer=None):
+ pool = None
+ pool_jobs = None
+
+ def open(self, analyzer=None, threads=4):
if ReusableIndex.index is not None:
self.index = ReusableIndex.index
else:
if ReusableIndex.index is not None:
self.index = ReusableIndex.index
else:
- Index.open(self,analyzer)
+ ReusableIndex.pool = ThreadPool(threads)
+ ReusableIndex.pool_jobs = []
+ Index.open(self, analyzer)
ReusableIndex.index = self.index
atexit.register(ReusableIndex.close_reusable)
ReusableIndex.index = self.index
atexit.register(ReusableIndex.close_reusable)
+ def index_book(self, *args, **kw):
+ job = ReusableIndex.pool.apply_async(Index.index_book, args, kw)
+ ReusableIndex.pool_jobs.append(job)
+
@staticmethod
def close_reusable():
if ReusableIndex.index is not None:
@staticmethod
def close_reusable():
if ReusableIndex.index is not None:
+ all_jobs = len(ReusableIndex.pool_jobs)
+ waited=1
+ for job in ReusableIndex.pool_jobs:
+ sys.stdout.write("\rWaiting for search index job: %d/%d..." %
+ job.wait()
+ waited+=1
+ print("Indexing done.")
+ ReusableIndex.pool.close()
+
ReusableIndex.index.optimize()
ReusableIndex.index.close()
ReusableIndex.index = None
ReusableIndex.index.optimize()
ReusableIndex.index.close()
ReusableIndex.index = None
-
+
def close(self):
pass
def close(self):
pass
+
class Search(IndexStore):
def __init__(self, default_field="content"):
IndexStore.__init__(self)
class Search(IndexStore):
def __init__(self, default_field="content"):
IndexStore.__init__(self)