OPDS with new SOLR interface.

fixed tests

diff --git a/apps/catalogue/models/book.py b/apps/catalogue/models/book.py
index 5bc1e10..980ad52 100644 (file)
--- a/apps/catalogue/models/book.py
+++ b/apps/catalogue/models/book.py
@@ -196,7 +196,7 @@ class Book(models.Model):
         try:
             index.index_book(self, book_info)
             if index_tags:
         try:
             index.index_book(self, book_info)
             if index_tags:

-                idx.index_tags()
+                index.index_tags()

             if commit:
                 index.index.commit()
         except Exception, e:
             if commit:
                 index.index.commit()
         except Exception, e:

diff --git a/apps/catalogue/test_utils.py b/apps/catalogue/test_utils.py
index 5e02619..1c888d7 100644 (file)
--- a/apps/catalogue/test_utils.py
+++ b/apps/catalogue/test_utils.py
@@ -9,6 +9,7 @@ from django.test import TestCase
 from django.test.utils import override_settings
 from slughifi import slughifi
 from librarian import WLURI
 from django.test.utils import override_settings
 from slughifi import slughifi
 from librarian import WLURI

+from django.conf import settings

 
 
 @override_settings(
 
 
 @override_settings(


@@ -21,6 +22,7 @@ from librarian import WLURI
             'default': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
             'permanent': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
         },
             'default': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
             'permanent': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'},
         },

+    SOLR = settings.SOLR_TEST,

 )
 class WLTestCase(TestCase):
     """
 )
 class WLTestCase(TestCase):
     """

diff --git a/apps/opds/tests/__init__.py b/apps/opds/tests/__init__.py
index d6e9ca0..856f2f2 100755 (executable)
--- a/apps/opds/tests/__init__.py
+++ b/apps/opds/tests/__init__.py
@@ -6,6 +6,7 @@ from catalogue.test_utils import (BookInfoStub, PersonStub, info_args,
         WLTestCase, get_fixture)
 from catalogue.models import Book
 from librarian import WLURI, XMLNamespace
         WLTestCase, get_fixture)
 from catalogue.models import Book
 from librarian import WLURI, XMLNamespace

+from search import Index, Search

 
 AtomNS = XMLNamespace("http://www.w3.org/2005/Atom")
 
 
 AtomNS = XMLNamespace("http://www.w3.org/2005/Atom")
 


@@ -14,6 +15,10 @@ class OpdsSearchTests(WLTestCase):
     """Tests search feed in OPDS.."""
     def setUp(self):
         WLTestCase.setUp(self)
     """Tests search feed in OPDS.."""
     def setUp(self):
         WLTestCase.setUp(self)

+        index = Index()
+        index.index.delete_all()
+        index.index.commit()
+

         with self.settings(NO_SEARCH_INDEX=False):
             self.do_doktora = Book.from_xml_file(
                 get_fixture('do-doktora.xml'))
         with self.settings(NO_SEARCH_INDEX=False):
             self.do_doktora = Book.from_xml_file(
                 get_fixture('do-doktora.xml'))


@@ -40,7 +45,7 @@ class OpdsSearchTests(WLTestCase):
         both = set([self.do_doktora, self.do_anusie])
         self.assert_finds('title=fraszka', both)
         self.assert_finds('title=fraszka', both)
         both = set([self.do_doktora, self.do_anusie])
         self.assert_finds('title=fraszka', both)
         self.assert_finds('title=fraszka', both)

-        self.assert_finds('q=title:fraszka', [self.do_doktora])
+        self.assert_finds('q=title:doktora', [self.do_doktora])

 
     def test_opds_search_author(self):
         """Search by author."""
 
     def test_opds_search_author(self):
         """Search by author."""

diff --git a/apps/opds/views.py b/apps/opds/views.py
index 297c212..f7658c2 100644 (file)
--- a/apps/opds/views.py
+++ b/apps/opds/views.py
@@ -18,7 +18,7 @@ from catalogue.models import Book, Tag
 
 from search.views import Search, SearchResult
 from lucene import Term, QueryWrapperFilter, TermQuery
 
 from search.views import Search, SearchResult
 from lucene import Term, QueryWrapperFilter, TermQuery

-
+import operator

 import logging
 import re
 
 import logging
 import re
 


@@ -328,7 +328,34 @@ class SearchFeed(AcquisitionFeed):
     description = u"Wyniki wyszukiwania na stronie WolneLektury.pl"
     title = u"Wyniki wyszukiwania"
 
     description = u"Wyniki wyszukiwania na stronie WolneLektury.pl"
     title = u"Wyniki wyszukiwania"
 

-    INLINE_QUERY_RE = re.compile(r"(author:(?P<author>[^ ]+)|title:(?P<title>[^ ]+)|categories:(?P<categories>[^ ]+)|description:(?P<description>[^ ]+))")
+    QUOTE_OR_NOT = r'(?:(?=["])"([^"]+)"|([^ ]+))'
+    INLINE_QUERY_RE = re.compile(
+        r"author:" + QUOTE_OR_NOT +
+        "|translator:" + QUOTE_OR_NOT +
+        "|title:" + QUOTE_OR_NOT +
+        "|categories:" + QUOTE_OR_NOT +
+        "|description:" + QUOTE_OR_NOT +
+        "|text:" + QUOTE_OR_NOT
+        )
+    MATCHES = {
+        'author': (0, 1),
+        'translator': (2, 3),
+        'title': (4, 5),
+        'categories': (6, 7),
+        'description': (8, 9),
+        'text': (10, 11),
+        }
+
+    PARAMS_TO_FIELDS = {
+        'author': 'authors',
+        'translator': 'translators',
+        #        'title': 'title',
+        'categories': 'tag_name_pl',
+        'description': 'text',
+        #        'text': 'text',
+        }
+
+    ATOM_PLACEHOLDER = re.compile(r"^{(atom|opds):\w+}$")

 
     def get_object(self, request):
         """
 
     def get_object(self, request):
         """


@@ -350,99 +377,62 @@ class SearchFeed(AcquisitionFeed):
 
         inline_criteria = re.findall(self.INLINE_QUERY_RE, query)
         if inline_criteria:
 
         inline_criteria = re.findall(self.INLINE_QUERY_RE, query)
         if inline_criteria:

-            def get_criteria(criteria, name, position):
-                e = filter(lambda el: el[0][0:len(name)] == name, criteria)
-                log.info("get_criteria: %s" % e)
-                if not e:
-                    return None
-                c = e[0][position]
-                log.info("get_criteria: %s" % c)
-                if c[0] == '"' and c[-1] == '"':
-                    c = c[1:-1]
-                    c = c.replace('+', ' ')
-                return c
-
-            author = get_criteria(inline_criteria, 'author', 1)
-            title = get_criteria(inline_criteria, 'title', 2)
-            translator = None
-            categories = get_criteria(inline_criteria, 'categories', 3)
-            query = get_criteria(inline_criteria, 'description', 4)
+            remains = re.sub(self.INLINE_QUERY_RE, '', query)
+            remains = re.sub(r'[ \t]+', ' ', remains)
+
+            def get_criteria(criteria, name):
+                for c in criteria:
+                    for p in self.MATCHES[name]:
+                        if c[p]:
+                            if p % 2 == 0:
+                                return c[p].replace('+', ' ')
+                            return c[p]
+                return None
+
+            criteria = dict(map(
+                lambda cn: (cn, get_criteria(inline_criteria, cn)),
+                ['author', 'translator', 'title', 'categories',
+                 'description', 'text']))
+            query = remains
+            # empty query and text set case?
+            log.debug("Inline query = [%s], criteria: %s" % (query, criteria))

         else:
         else:

-            author = request.GET.get('author', '')
-            title = request.GET.get('title', '')
-            translator = request.GET.get('translator', '')
+            def remove_dump_data(val):
+                """Some clients don't get opds placeholders and just send them."""
+                if self.ATOM_PLACEHOLDER.match(val):
+                    return ''
+                return val

 
 

-            # Our client didn't handle the opds placeholders
-            if author == '{atom:author}': author = ''
-            if title == '{atom:title}': title = ''
-            if translator == '{atom:contributor}': translator = ''
-            categories = None
-            fuzzy = False
+            criteria = dict([(cn, remove_dump_data(request.GET.get(cn, '')))
+                        for cn in self.MATCHES.keys()])
+            # query is set above.
+            log.debug("Inline query = [%s], criteria: %s" % (query, criteria))

 
         srch = Search()
 
         srch = Search()

-        hint = srch.hint()

 
 

-        # Scenario 1: full search terms provided.
-        # Use auxiliarry information to narrow it and make it better.
+        book_hit_filter = srch.index.Q(book_id__any=True)
+        filters = [book_hit_filter] + [srch.index.Q(
+            **{self.PARAMS_TO_FIELDS.get(cn, cn): criteria[cn]}
+            ) for cn in self.MATCHES.keys() if cn in criteria
+            if criteria[cn]]
+

         if query:
         if query:

-            filters = []
-
-            if author:
-                log.info("narrow to author %s" % author)
-                hint.tags(srch.search_tags(srch.make_phrase(srch.get_tokens(author, field='authors'), field='authors'),
-                                            filt=srch.term_filter(Term('tag_category', 'author'))))
-
-            if translator:
-                log.info("filter by translator %s" % translator)
-                filters.append(QueryWrapperFilter(
-                    srch.make_phrase(srch.get_tokens(translator, field='translators'),
-                                     field='translators')))
-
-            if categories:
-                filters.append(QueryWrapperFilter(
-                    srch.make_phrase(srch.get_tokens(categories, field="tag_name_pl"),
-                                     field='tag_name_pl')))
-
-            flt = srch.chain_filters(filters)
-            if title:
-                log.info("hint by book title %s" % title)
-                q = srch.make_phrase(srch.get_tokens(title, field='title'), field='title')
-                hint.books(*srch.search_books(q, filt=flt))
-
-            toks = srch.get_tokens(query)
-            log.info("tokens for query: %s" % toks)
-
-            results = SearchResult.aggregate(srch.search_perfect_book(toks, fuzzy=fuzzy, hint=hint),
-                srch.search_perfect_parts(toks, fuzzy=fuzzy, hint=hint),
-                srch.search_everywhere(toks, fuzzy=fuzzy, hint=hint))
-            results.sort(reverse=True)
-            books = []
-            for r in results:
-                try:
-                    books.append(r.book)
-                except Book.DoesNotExist:
-                    pass
-            log.info("books: %s" % books)
-            return books
+            q = srch.index.query(
+                reduce(operator.or_,
+                       [srch.index.Q(**{self.PARAMS_TO_FIELDS.get(cn, cn): query})
+                        for cn in self.MATCHES.keys()],
+                srch.index.Q()))

         else:
         else:

-            # Scenario 2: since we no longer have to figure out what the query term means to the user,
-            # we can just use filters and not the Hint class.
-            filters = []
-
-            fields = {
-                'author': author,
-                'translators': translator,
-                'title': title
-                }
-
-            for fld, q in fields.items():
-                if q:
-                    filters.append(QueryWrapperFilter(
-                        srch.make_phrase(srch.get_tokens(q, field=fld), field=fld)))
-
-            flt = srch.chain_filters(filters)
-            books = srch.search_books(TermQuery(Term('is_book', 'true')), filt=flt)
-            return books
+            q = srch.index.query(srch.index.Q())
+
+        q = srch.apply_filters(q, filters).field_limit(score=True, fields=['book_id'])
+        results = q.execute()
+
+        book_scores = dict([(r['book_id'], r['score']) for r in results])
+        books = Book.objects.filter(id__in=set([r['book_id'] for r in results]))
+        books = list(books)
+        books.sort(reverse=True, key=lambda book: book_scores[book.id])
+        return books

 
     def get_link(self, query):
         return "%s?q=%s" % (reverse('search'), query)
 
     def get_link(self, query):
         return "%s?q=%s" % (reverse('search'), query)

diff --git a/apps/search/index.py b/apps/search/index.py
index 557f404..556bbfe 100644 (file)
--- a/apps/search/index.py
+++ b/apps/search/index.py
@@ -18,6 +18,7 @@ import sunburnt
 import custom
 import operator
 
 import custom
 import operator
 

+log = logging.getLogger('search')

 
 class SolrIndex(object):
     def __init__(self, mode=None):
 
 class SolrIndex(object):
     def __init__(self, mode=None):


@@ -240,7 +241,7 @@ class Index(SolrIndex):
             self.remove_book(book, remove_snippets=False)
 
         book_doc = self.create_book_doc(book)
             self.remove_book(book, remove_snippets=False)
 
         book_doc = self.create_book_doc(book)

-        meta_fields = self.extract_metadata(book, book_info, dc_only=['source_name', 'authors', 'title'])
+        meta_fields = self.extract_metadata(book, book_info, dc_only=['source_name', 'authors', 'translators', 'title'])

         # let's not index it - it's only used for extracting publish date
         if 'source_name' in meta_fields:
             del meta_fields['source_name']
         # let's not index it - it's only used for extracting publish date
         if 'source_name' in meta_fields:
             del meta_fields['source_name']


@@ -256,6 +257,7 @@ class Index(SolrIndex):
             'authors': meta_fields['authors'],
             'published_date': meta_fields['published_date']
             }
             'authors': meta_fields['authors'],
             'published_date': meta_fields['published_date']
             }

+

         if 'translators' in meta_fields:
             book_fields['translators'] = meta_fields['translators']
 
         if 'translators' in meta_fields:
             book_fields['translators'] = meta_fields['translators']
 


@@ -791,6 +793,7 @@ class Search(SolrIndex):
         modal - applies to boolean query
         fuzzy - should the query by fuzzy.
         """
         modal - applies to boolean query
         fuzzy - should the query by fuzzy.
         """

+        if query is None: query = ''

         q = self.index.Q()
         q = reduce(modal, map(lambda s: self.index.Q(**{field: s}),
                         query.split(r" ")), q)
         q = self.index.Q()
         q = reduce(modal, map(lambda s: self.index.Q(**{field: s}),
                         query.split(r" ")), q)

diff --git a/wolnelektury/settings/basic.py b/wolnelektury/settings/basic.py
index e5ba5cd..dea4b42 100644 (file)
--- a/wolnelektury/settings/basic.py
+++ b/wolnelektury/settings/basic.py
@@ -22,7 +22,7 @@ DATABASES = {
 }
 
 SOLR = "http://localhost:8983/solr/wl/"
 }
 
 SOLR = "http://localhost:8983/solr/wl/"

-
+SOLR_TEST = "http://localhost:8983/solr/wl_test/"

 
 # Local time zone for this installation. Choices can be found here:
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name
 
 # Local time zone for this installation. Choices can be found here:
 # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name