Merge branch 'master' into rwd
[wolnelektury.git] / apps / search / views.py
index ec8275b..9a0b469 100644 (file)
@@ -3,7 +3,6 @@
 from django.conf import settings
 from django.shortcuts import render_to_response, get_object_or_404
 from django.template import RequestContext
-from django.contrib.auth.decorators import login_required
 from django.views.decorators import cache
 from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponsePermanentRedirect
 from django.utils.translation import ugettext as _
@@ -12,16 +11,12 @@ from catalogue.utils import split_tags
 from catalogue.models import Book, Tag, Fragment
 from pdcounter.models import Author as PDCounterAuthor, BookStub as PDCounterBook
 from catalogue.views import JSONResponse
-from search import Search, JVM, SearchResult
-from lucene import StringReader
+from search import Search, SearchResult
 from suggest.forms import PublishingSuggestForm
-from time import sleep
 import re
 #import enchant
 import json
 
-#dictionary = enchant.Dict('en_US')
-
 
 def match_word_re(word):
     if 'sqlite' in settings.DATABASES['default']['ENGINE']:
@@ -30,61 +25,47 @@ def match_word_re(word):
         return "[[:<:]]%s[[:>:]]" % word
 
 
-def did_you_mean(query, tokens):
-    change = {}
-    for t in tokens:
-        authors = Tag.objects.filter(category='author', name__iregex=match_word_re(t))
-        if len(authors) > 0:
-            continue
-
-        if False:
-            if not dictionary.check(t):
-                try:
-                    change_to = dictionary.suggest(t)[0].lower()
-                    if change_to != t.lower():
-                        change[t] = change_to
-                except IndexError:
-                    pass
-
-    if change == {}:
-        return None
-
-    for frm, to in change.items():
-        query = query.replace(frm, to)
+query_syntax_chars = re.compile(r"[\\/*:(){}]")
 
-    return query
+
+def remove_query_syntax_chars(query, replace=' '):
+    return query_syntax_chars.sub(' ', query)
 
 
-JVM.attachCurrentThread()
-_search = None
+def did_you_mean(query, tokens):
+    return query
+    # change = {}
+    # for t in tokens:
+    #     authors = Tag.objects.filter(category='author', name__iregex=match_word_re(t))
+    #     if len(authors) > 0:
+    #         continue
 
+    #     if False:
+    #         if not dictionary.check(t):
+    #             try:
+    #                 change_to = dictionary.suggest(t)[0].lower()
+    #                 if change_to != t.lower():
+    #                     change[t] = change_to
+    #             except IndexError:
+    #                 pass
 
-def get_search():
-    global _search
+    # if change == {}:
+    #     return None
 
-    while _search is False:
-        sleep(1)
+    # for frm, to in change.items():
+    #     query = query.replace(frm, to)
 
-    if _search is None:
-        _search = False
-        _search = Search()
-    return _search
+    # return query
 
 
 def hint(request):
     prefix = request.GET.get('term', '')
     if len(prefix) < 2:
         return JSONResponse([])
-    JVM.attachCurrentThread()
 
-    search = get_search()
-    hint = search.hint()
-    try:
-        tags = request.GET.get('tags', '')
-        hint.tags(Tag.get_tag_list(tags))
-    except:
-        pass
+    prefix = remove_query_syntax_chars(prefix)
 
+    search = Search()
     # tagi beda ograniczac tutaj
     # ale tagi moga byc na ksiazce i na fragmentach
     # jezeli tagi dot tylko ksiazki, to wazne zeby te nowe byly w tej samej ksiazce
@@ -93,7 +74,6 @@ def hint(request):
     tags = search.hint_tags(prefix, pdcounter=True)
     books = search.hint_books(prefix)
 
-    
     def is_dupe(tag):
         if isinstance(tag, PDCounterAuthor):
             if filter(lambda t: t.slug == tag.slug and t != tag, tags):
@@ -126,64 +106,56 @@ def hint(request):
                             content_type="application/json; charset=utf-8")
     else:
         return JSONResponse(data)
-            
 
 
 def main(request):
     results = {}
-    JVM.attachCurrentThread()  # where to put this?
 
     results = None
     query = None
-    fuzzy = False #0.8
-
-    query = request.GET.get('q','')
-    # book_id = request.GET.get('book', None)
-    # book = None
-    # if book_id is not None:
-    #     book = get_object_or_404(Book, id=book_id)
 
-    # hint = search.hint()
-    # try:
-    #     tag_list = Tag.get_tag_list(tags)
-    # except:
-    #     tag_list = []
+    query = request.GET.get('q', '')
 
     if len(query) < 2:
-        return render_to_response('catalogue/search_too_short.html', {'prefix': query},
-                                  context_instance=RequestContext(request))
-
-    search = get_search()
-    # hint.tags(tag_list)
-    # if book:
-    #     hint.books(book)
-    tags = search.hint_tags(query, pdcounter=True, prefix=False, fuzzy=fuzzy)
+        return render_to_response('catalogue/search_too_short.html',
+                                  {'prefix': query},
+            context_instance=RequestContext(request))
+
+    query = remove_query_syntax_chars(query)
+    
+    search = Search()
+
+    theme_terms = search.index.analyze(text=query, field="themes_pl") \
+        + search.index.analyze(text=query, field="themes")
+
+            # change hints
+    tags = search.hint_tags(query, pdcounter=True, prefix=False)
     tags = split_tags(tags)
 
-    toks = StringReader(query)
-    tokens_cache = {}
+    author_results = search.search_phrase(query, 'authors', book=True)
+    translator_results = search.search_phrase(query, 'translators', book=True)
 
-    author_results = search.search_phrase(toks, 'authors', fuzzy=fuzzy, tokens_cache=tokens_cache)
-    title_results = search.search_phrase(toks, 'title', fuzzy=fuzzy, tokens_cache=tokens_cache)
+    title_results = search.search_phrase(query, 'title', book=True)
 
     # Boost main author/title results with mixed search, and save some of its results for end of list.
     # boost author, title results
-    author_title_mixed = search.search_some(toks, ['authors', 'title', 'tags'], fuzzy=fuzzy, tokens_cache=tokens_cache)
+    author_title_mixed = search.search_some(query, ['authors', 'translators', 'title', 'tags'], query_terms=theme_terms)
     author_title_rest = []
+
     for b in author_title_mixed:
-        bks = filter(lambda ba: ba.book_id == b.book_id, author_results + title_results)
-        for b2 in bks:
+        also_in_mixed = filter(lambda ba: ba.book_id == b.book_id, author_results + translator_results + title_results)
+        for b2 in also_in_mixed:
             b2.boost *= 1.1
-        if bks is []:
+        if also_in_mixed is []:
             author_title_rest.append(b)
 
     # Do a phrase search but a term search as well - this can give us better snippets then search_everywhere,
     # Because the query is using only one field.
     text_phrase = SearchResult.aggregate(
-        search.search_phrase(toks, 'content', fuzzy=fuzzy, tokens_cache=tokens_cache, snippets=True, book=False, slop=4),
-        search.search_some(toks, ['content'], tokens_cache=tokens_cache, snippets=True, book=False))
+        search.search_phrase(query, 'text', snippets=True, book=False),
+        search.search_some(query, ['text'], snippets=True, book=False, query_terms=theme_terms))
 
-    everywhere = search.search_everywhere(toks, fuzzy=fuzzy, tokens_cache=tokens_cache)
+    everywhere = search.search_everywhere(query, query_terms=theme_terms)
 
     def already_found(results):
         def f(e):
@@ -194,23 +166,25 @@ def main(request):
                     return True
             return False
         return f
-    f = already_found(author_results + title_results + text_phrase)
+    f = already_found(author_results + translator_results + title_results + text_phrase)
     everywhere = filter(lambda x: not f(x), everywhere)
 
     author_results = SearchResult.aggregate(author_results)
+    translator_results = SearchResult.aggregate(translator_results)
     title_results = SearchResult.aggregate(title_results)
 
     everywhere = SearchResult.aggregate(everywhere, author_title_rest)
 
-    for res in [author_results, title_results, text_phrase, everywhere]:
+    for field, res in [('authors', author_results),
+                       ('translators', translator_results),
+                       ('title', title_results),
+                       ('text', text_phrase),
+                       ('text', everywhere)]:
         res.sort(reverse=True)
         for r in res:
-            for h in r.hits:
-                h['snippets'] = map(lambda s:
-                                    re.subn(r"(^[ \t\n]+|[ \t\n]+$)", u"",
-                                            re.subn(r"[ \t\n]*\n[ \t\n]*", u"\n", s)[0])[0], h['snippets'])
+            search.get_snippets(r, query, field, 3)
 
-    suggestion = did_you_mean(query, search.get_tokens(toks, field="SIMPLE"))
+    suggestion = u''
 
     def ensure_exists(r):
         try:
@@ -219,22 +193,25 @@ def main(request):
             return False
 
     author_results = filter(ensure_exists, author_results)
+    translator_results = filter(ensure_exists, translator_results)
     title_results = filter(ensure_exists, title_results)
     text_phrase = filter(ensure_exists, text_phrase)
     everywhere = filter(ensure_exists, everywhere)
 
-    results = author_results + title_results + text_phrase + everywhere
+    results = author_results + translator_results + title_results + text_phrase + everywhere
     # ensure books do exists & sort them
-    results.sort(reverse=True)
-
-    if len(results) == 1:
-        fragment_hits = filter(lambda h: 'fragment' in h, results[0].hits)
-        if len(fragment_hits) == 1:
-            #anchor = fragment_hits[0]['fragment']
-            #frag = Fragment.objects.get(anchor=anchor)
-            return HttpResponseRedirect(fragment_hits[0]['fragment'].get_absolute_url())
-        return HttpResponseRedirect(results[0].book.get_absolute_url())
-    elif len(results) == 0:
+    for res in (author_results, translator_results, title_results, text_phrase, everywhere):
+        res.sort(reverse=True)
+
+    # We don't want to redirect to book text, but rather display result page even with one result.
+    # if len(results) == 1:
+    #     fragment_hits = filter(lambda h: 'fragment' in h, results[0].hits)
+    #     if len(fragment_hits) == 1:
+    #         #anchor = fragment_hits[0]['fragment']
+    #         #frag = Fragment.objects.get(anchor=anchor)
+    #         return HttpResponseRedirect(fragment_hits[0]['fragment'].get_absolute_url())
+    #     return HttpResponseRedirect(results[0].book.get_absolute_url())
+    if len(results) == 0:
         form = PublishingSuggestForm(initial={"books": query + ", "})
         return render_to_response('catalogue/search_no_hits.html',
                                   {'tags': tags,
@@ -246,9 +223,10 @@ def main(request):
     return render_to_response('catalogue/search_multiple_hits.html',
                               {'tags': tags,
                                'prefix': query,
-                               'results': { 'author': author_results,
-                                            'title': title_results,
-                                            'content': text_phrase,
-                                            'other': everywhere},
+                               'results': {'author': author_results,
+                                           'translator': translator_results,
+                                           'title': title_results,
+                                           'content': text_phrase,
+                                           'other': everywhere},
                                'did_you_mean': suggestion},
         context_instance=RequestContext(request))