class SearchResult(object):
- def __init__(self, searcher, scoreDocs, score=None, how_found=None, snippets=None):
+ def __init__(self, searcher, scoreDocs, score=None, how_found=None, snippets=None, searched=None, tokens_cache=None):
+ if tokens_cache is None: tokens_cache = {}
+
if score:
- self.score = score
+ self._score = score
else:
- self.score = scoreDocs.score
+ self._score = scoreDocs.score
+
+ self.boost = 1.0
self._hits = []
self.hits = None # processed hits
self._hits.append(hit)
+ self.searcher = searcher
+ self.searched = searched
+ self.tokens_cache = tokens_cache
+
+ @property
+ def score(self):
+ return self._score * self.boost
+
def merge(self, other):
if self.book_id != other.book_id:
raise ValueError("this search result is or book %d; tried to merge with %d" % (self.book_id, other.book_id))
for f in frags:
frag = catalogue.models.Fragment.objects.get(anchor=f[FRAGMENT])
+
+ # Figure out if we were searching for a token matching some word in theme name.
+ themes = frag.tags.filter(category='theme')
+ themes_hit = []
+ if self.searched is not None:
+ tokens = self.searcher.get_tokens(self.searched, 'POLISH', tokens_cache=self.tokens_cache)
+ for theme in themes:
+ name_tokens = self.searcher.get_tokens(theme.name, 'POLISH')
+ for t in tokens:
+ if name_tokens.index(t):
+ if not theme in themes_hit:
+ themes_hit.append(theme)
+ break
+
m = {'score': f[SCORE],
'fragment': frag,
'section_number': f[POSITION][POSITION_INDEX] + 1,
- 'themes': frag.tags.filter(category='theme')
+ 'themes': themes,
+ 'themes_hit': themes_hit
}
m.update(f[OTHER])
hits.append(m)
bks.append(catalogue.models.Book.objects.get(id=doc.get("book_id")))
return (bks, tops.totalHits)
- def get_tokens(self, searched, field='content'):
+ def get_tokens(self, searched, field='content', cached=None):
"""returns tokens analyzed by a proper (for a field) analyzer
argument can be: StringReader, string/unicode, or tokens. In the last case
they will just be returned (so we can reuse tokens, if we don't change the analyzer)
"""
+ if cached is not None and field in cached:
+ return cached[field]
+
if isinstance(searched, str) or isinstance(searched, unicode):
searched = StringReader(searched)
elif isinstance(searched, list):
while tokens.incrementToken():
cta = tokens.getAttribute(CharTermAttribute.class_)
toks.append(cta.toString())
+
+ if cached is not None:
+ cached[field] = toks
+
return toks
def fuzziness(self, fuzzy):
q.add(BooleanClause(term, modal))
return q
- # def content_query(self, query):
- # return BlockJoinQuery(query, self.parent_filter,
- # BlockJoinQuery.ScoreMode.Total)
+ def search_phrase(self, searched, field, book=True, max_results=20, fuzzy=False,
+ filters=None, tokens_cache=None, boost=None):
+ if filters is None: filters = []
+ if tokens_cache is None: tokens_cache = {}
+
+ tokens = self.get_tokens(searched, field, cached=tokens_cache)
+
+ query = self.make_phrase(tokens, field=field, fuzzy=fuzzy)
+ if book:
+ filters.append(self.term_filter(Term('is_book', 'true')))
+ top = self.searcher.search(query, self.chain_filters(filters), max_results)
+
+ return [SearchResult(self.searcher, found) for found in top.scoreDocs]
+
+ def search_some(self, searched, fields, book=True, max_results=20, fuzzy=False,
+ filters=None, tokens_cache=None, boost=None):
+ if filters is None: filters = []
+ if tokens_cache is None: tokens_cache = {}
+
+ if book:
+ filters.append(self.term_filter(Term('is_book', 'true')))
+
+ query = BooleanQuery()
+
+ for fld in fields:
+ tokens = self.get_tokens(searched, fld, cached=tokens_cache)
+
+ query.add(BooleanClause(self.make_term_query(tokens, field=fld,
+ fuzzy=fuzzy), BooleanClause.Occur.SHOULD))
+
+ top = self.searcher.search(query, self.chain_filters(filters), max_results)
+
+ return [SearchResult(self.searcher, found, searched=searched, tokens_cache=tokens_cache) for found in top.scoreDocs]
def search_perfect_book(self, searched, max_results=20, fuzzy=False, hint=None):
"""
def search_perfect_parts(self, searched, max_results=20, fuzzy=False, hint=None):
"""
- Search for book parts which containt a phrase perfectly matching (with a slop of 2, default for make_phrase())
+ Search for book parts which contains a phrase perfectly matching (with a slop of 2, default for make_phrase())
some part/fragment of the book.
"""
qrys = [self.make_phrase(self.get_tokens(searched), field=fld, fuzzy=fuzzy) for fld in ['content']]
return books
- def search_everywhere(self, searched, max_results=20, fuzzy=False, hint=None):
+ def search_everywhere(self, searched, max_results=20, fuzzy=False, hint=None, tokens_cache=None):
"""
Tries to use search terms to match different fields of book (or its parts).
E.g. one word can be an author survey, another be a part of the title, and the rest
are some words from third chapter.
"""
+ if tokens_cache is None: tokens_cache = {}
books = []
only_in = None
# content only query : themes x content
q = BooleanQuery()
- tokens_pl = self.get_tokens(searched, field='content')
- tokens = self.get_tokens(searched, field='SIMPLE')
+ tokens_pl = self.get_tokens(searched, field='content', cached=tokens_cache)
+ tokens = self.get_tokens(searched, field='SIMPLE', cached=tokens_cache)
# only search in themes when we do not already filter by themes
if hint is None or hint.just_search_in(['themes']) != []:
Chains a filter list together
"""
filters = filter(lambda x: x is not None, filters)
- if not filters:
+ if not filters or filters is []:
return None
chf = ChainedFilter(JArray('object')(filters, Filter), op)
return chf
hint.books(book)
toks = StringReader(query)
+ tokens_cache = {}
fuzzy = 'fuzzy' in request.GET
if fuzzy:
fuzzy = 0.7
- results = SearchResult.aggregate(srch.search_perfect_book(toks, fuzzy=fuzzy, hint=hint),
- srch.search_book(toks, fuzzy=fuzzy, hint=hint),
- srch.search_perfect_parts(toks, fuzzy=fuzzy, hint=hint),
- srch.search_everywhere(toks, fuzzy=fuzzy, hint=hint))
-
- for r in results:
- r.process_hits()
-
- results.sort(reverse=True)
+ author_results = srch.search_phrase(toks, 'authors', fuzzy=fuzzy, tokens_cache=tokens_cache)
+ title_results = srch.search_phrase(toks, 'title', fuzzy=fuzzy, tokens_cache=tokens_cache)
+
+ # Boost main author/title results with mixed search, and save some of its results for end of list.
+ # boost author, title results
+ author_title_mixed = srch.search_some(toks, ['authors', 'title', 'tags'], fuzzy=fuzzy, tokens_cache=tokens_cache)
+ author_title_rest = []
+ for b in author_title_mixed:
+ bks = filter(lambda ba: ba.book_id == b.book_id, author_results + title_results)
+ for b2 in bks:
+ b2.boost *= 1.1
+ if bks is []:
+ author_title_rest.append(b)
+
+ text_phrase = SearchResult.aggregate(srch.search_phrase(toks, 'content', fuzzy=fuzzy, tokens_cache=tokens_cache))
+ [r.process_hits() for r in text_phrase]
+
+ everywhere = SearchResult.aggregate(srch.search_everywhere(toks, fuzzy=fuzzy, tokens_cache=tokens_cache), author_title_rest)
+ [r.process_hits() for r in everywhere]
- for r in results:
- print "-----"
- for h in r.hits:
- print "- %s" % h
+ for res in [author_results, title_results, text_phrase, everywhere]:
+ res.sort(reverse=True)
- # Did you mean?
suggestion = did_you_mean(query, srch.get_tokens(toks, field="SIMPLE"))
+ results = author_results + title_results + text_phrase + everywhere
+ results.sort(reverse=True)
+
if len(results) == 1:
if len(results[0].hits) == 0:
return HttpResponseRedirect(results[0].book.get_absolute_url())
return render_to_response('catalogue/search_multiple_hits.html',
{'tags': tag_list,
'prefix': query,
- 'results': results,
+ 'results': { 'author': author_results,
+ 'title': title_results,
+ 'content': text_phrase,
+ 'other': everywhere},
'did_you_mean': suggestion},
context_instance=RequestContext(request))
fnp / wolnelektury.git / commitdiff