From: Marcin Koziej Date: Tue, 11 Sep 2012 15:01:56 +0000 (+0200) Subject: OPDS with new SOLR interface. X-Git-Url: https://git.mdrn.pl/wolnelektury.git/commitdiff_plain/f3acfae26c01f08232af6f4b48e9a2ea54a0c2da?ds=sidebyside;hp=ac6111b68183ba7da48252eb0f7389b24ede20a9 OPDS with new SOLR interface. fixed tests --- diff --git a/apps/catalogue/models/book.py b/apps/catalogue/models/book.py index 5bc1e1021..980ad5269 100644 --- a/apps/catalogue/models/book.py +++ b/apps/catalogue/models/book.py @@ -196,7 +196,7 @@ class Book(models.Model): try: index.index_book(self, book_info) if index_tags: - idx.index_tags() + index.index_tags() if commit: index.index.commit() except Exception, e: diff --git a/apps/catalogue/test_utils.py b/apps/catalogue/test_utils.py index 5e02619f9..1c888d760 100644 --- a/apps/catalogue/test_utils.py +++ b/apps/catalogue/test_utils.py @@ -9,6 +9,7 @@ from django.test import TestCase from django.test.utils import override_settings from slughifi import slughifi from librarian import WLURI +from django.conf import settings @override_settings( @@ -21,6 +22,7 @@ from librarian import WLURI 'default': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}, 'permanent': {'BACKEND': 'django.core.cache.backends.dummy.DummyCache'}, }, + SOLR = settings.SOLR_TEST, ) class WLTestCase(TestCase): """ diff --git a/apps/opds/tests/__init__.py b/apps/opds/tests/__init__.py index d6e9ca05a..856f2f2e5 100755 --- a/apps/opds/tests/__init__.py +++ b/apps/opds/tests/__init__.py @@ -6,6 +6,7 @@ from catalogue.test_utils import (BookInfoStub, PersonStub, info_args, WLTestCase, get_fixture) from catalogue.models import Book from librarian import WLURI, XMLNamespace +from search import Index, Search AtomNS = XMLNamespace("http://www.w3.org/2005/Atom") @@ -14,6 +15,10 @@ class OpdsSearchTests(WLTestCase): """Tests search feed in OPDS..""" def setUp(self): WLTestCase.setUp(self) + index = Index() + index.index.delete_all() + index.index.commit() + with self.settings(NO_SEARCH_INDEX=False): self.do_doktora = Book.from_xml_file( get_fixture('do-doktora.xml')) @@ -40,7 +45,7 @@ class OpdsSearchTests(WLTestCase): both = set([self.do_doktora, self.do_anusie]) self.assert_finds('title=fraszka', both) self.assert_finds('title=fraszka', both) - self.assert_finds('q=title:fraszka', [self.do_doktora]) + self.assert_finds('q=title:doktora', [self.do_doktora]) def test_opds_search_author(self): """Search by author.""" diff --git a/apps/opds/views.py b/apps/opds/views.py index 297c2120a..f7658c22b 100644 --- a/apps/opds/views.py +++ b/apps/opds/views.py @@ -18,7 +18,7 @@ from catalogue.models import Book, Tag from search.views import Search, SearchResult from lucene import Term, QueryWrapperFilter, TermQuery - +import operator import logging import re @@ -328,7 +328,34 @@ class SearchFeed(AcquisitionFeed): description = u"Wyniki wyszukiwania na stronie WolneLektury.pl" title = u"Wyniki wyszukiwania" - INLINE_QUERY_RE = re.compile(r"(author:(?P[^ ]+)|title:(?P[^ ]+)|categories:(?P<categories>[^ ]+)|description:(?P<description>[^ ]+))") + QUOTE_OR_NOT = r'(?:(?=["])"([^"]+)"|([^ ]+))' + INLINE_QUERY_RE = re.compile( + r"author:" + QUOTE_OR_NOT + + "|translator:" + QUOTE_OR_NOT + + "|title:" + QUOTE_OR_NOT + + "|categories:" + QUOTE_OR_NOT + + "|description:" + QUOTE_OR_NOT + + "|text:" + QUOTE_OR_NOT + ) + MATCHES = { + 'author': (0, 1), + 'translator': (2, 3), + 'title': (4, 5), + 'categories': (6, 7), + 'description': (8, 9), + 'text': (10, 11), + } + + PARAMS_TO_FIELDS = { + 'author': 'authors', + 'translator': 'translators', + # 'title': 'title', + 'categories': 'tag_name_pl', + 'description': 'text', + # 'text': 'text', + } + + ATOM_PLACEHOLDER = re.compile(r"^{(atom|opds):\w+}$") def get_object(self, request): """ @@ -350,99 +377,62 @@ class SearchFeed(AcquisitionFeed): inline_criteria = re.findall(self.INLINE_QUERY_RE, query) if inline_criteria: - def get_criteria(criteria, name, position): - e = filter(lambda el: el[0][0:len(name)] == name, criteria) - log.info("get_criteria: %s" % e) - if not e: - return None - c = e[0][position] - log.info("get_criteria: %s" % c) - if c[0] == '"' and c[-1] == '"': - c = c[1:-1] - c = c.replace('+', ' ') - return c - - author = get_criteria(inline_criteria, 'author', 1) - title = get_criteria(inline_criteria, 'title', 2) - translator = None - categories = get_criteria(inline_criteria, 'categories', 3) - query = get_criteria(inline_criteria, 'description', 4) + remains = re.sub(self.INLINE_QUERY_RE, '', query) + remains = re.sub(r'[ \t]+', ' ', remains) + + def get_criteria(criteria, name): + for c in criteria: + for p in self.MATCHES[name]: + if c[p]: + if p % 2 == 0: + return c[p].replace('+', ' ') + return c[p] + return None + + criteria = dict(map( + lambda cn: (cn, get_criteria(inline_criteria, cn)), + ['author', 'translator', 'title', 'categories', + 'description', 'text'])) + query = remains + # empty query and text set case? + log.debug("Inline query = [%s], criteria: %s" % (query, criteria)) else: - author = request.GET.get('author', '') - title = request.GET.get('title', '') - translator = request.GET.get('translator', '') + def remove_dump_data(val): + """Some clients don't get opds placeholders and just send them.""" + if self.ATOM_PLACEHOLDER.match(val): + return '' + return val - # Our client didn't handle the opds placeholders - if author == '{atom:author}': author = '' - if title == '{atom:title}': title = '' - if translator == '{atom:contributor}': translator = '' - categories = None - fuzzy = False + criteria = dict([(cn, remove_dump_data(request.GET.get(cn, ''))) + for cn in self.MATCHES.keys()]) + # query is set above. + log.debug("Inline query = [%s], criteria: %s" % (query, criteria)) srch = Search() - hint = srch.hint() - # Scenario 1: full search terms provided. - # Use auxiliarry information to narrow it and make it better. + book_hit_filter = srch.index.Q(book_id__any=True) + filters = [book_hit_filter] + [srch.index.Q( + **{self.PARAMS_TO_FIELDS.get(cn, cn): criteria[cn]} + ) for cn in self.MATCHES.keys() if cn in criteria + if criteria[cn]] + if query: - filters = [] - - if author: - log.info("narrow to author %s" % author) - hint.tags(srch.search_tags(srch.make_phrase(srch.get_tokens(author, field='authors'), field='authors'), - filt=srch.term_filter(Term('tag_category', 'author')))) - - if translator: - log.info("filter by translator %s" % translator) - filters.append(QueryWrapperFilter( - srch.make_phrase(srch.get_tokens(translator, field='translators'), - field='translators'))) - - if categories: - filters.append(QueryWrapperFilter( - srch.make_phrase(srch.get_tokens(categories, field="tag_name_pl"), - field='tag_name_pl'))) - - flt = srch.chain_filters(filters) - if title: - log.info("hint by book title %s" % title) - q = srch.make_phrase(srch.get_tokens(title, field='title'), field='title') - hint.books(*srch.search_books(q, filt=flt)) - - toks = srch.get_tokens(query) - log.info("tokens for query: %s" % toks) - - results = SearchResult.aggregate(srch.search_perfect_book(toks, fuzzy=fuzzy, hint=hint), - srch.search_perfect_parts(toks, fuzzy=fuzzy, hint=hint), - srch.search_everywhere(toks, fuzzy=fuzzy, hint=hint)) - results.sort(reverse=True) - books = [] - for r in results: - try: - books.append(r.book) - except Book.DoesNotExist: - pass - log.info("books: %s" % books) - return books + q = srch.index.query( + reduce(operator.or_, + [srch.index.Q(**{self.PARAMS_TO_FIELDS.get(cn, cn): query}) + for cn in self.MATCHES.keys()], + srch.index.Q())) else: - # Scenario 2: since we no longer have to figure out what the query term means to the user, - # we can just use filters and not the Hint class. - filters = [] - - fields = { - 'author': author, - 'translators': translator, - 'title': title - } - - for fld, q in fields.items(): - if q: - filters.append(QueryWrapperFilter( - srch.make_phrase(srch.get_tokens(q, field=fld), field=fld))) - - flt = srch.chain_filters(filters) - books = srch.search_books(TermQuery(Term('is_book', 'true')), filt=flt) - return books + q = srch.index.query(srch.index.Q()) + + q = srch.apply_filters(q, filters).field_limit(score=True, fields=['book_id']) + results = q.execute() + + book_scores = dict([(r['book_id'], r['score']) for r in results]) + books = Book.objects.filter(id__in=set([r['book_id'] for r in results])) + books = list(books) + books.sort(reverse=True, key=lambda book: book_scores[book.id]) + return books def get_link(self, query): return "%s?q=%s" % (reverse('search'), query) diff --git a/apps/search/index.py b/apps/search/index.py index 557f4045c..556bbfe5c 100644 --- a/apps/search/index.py +++ b/apps/search/index.py @@ -18,6 +18,7 @@ import sunburnt import custom import operator +log = logging.getLogger('search') class SolrIndex(object): def __init__(self, mode=None): @@ -240,7 +241,7 @@ class Index(SolrIndex): self.remove_book(book, remove_snippets=False) book_doc = self.create_book_doc(book) - meta_fields = self.extract_metadata(book, book_info, dc_only=['source_name', 'authors', 'title']) + meta_fields = self.extract_metadata(book, book_info, dc_only=['source_name', 'authors', 'translators', 'title']) # let's not index it - it's only used for extracting publish date if 'source_name' in meta_fields: del meta_fields['source_name'] @@ -256,6 +257,7 @@ class Index(SolrIndex): 'authors': meta_fields['authors'], 'published_date': meta_fields['published_date'] } + if 'translators' in meta_fields: book_fields['translators'] = meta_fields['translators'] @@ -791,6 +793,7 @@ class Search(SolrIndex): modal - applies to boolean query fuzzy - should the query by fuzzy. """ + if query is None: query = '' q = self.index.Q() q = reduce(modal, map(lambda s: self.index.Q(**{field: s}), query.split(r" ")), q) diff --git a/wolnelektury/settings/basic.py b/wolnelektury/settings/basic.py index e5ba5cd16..dea4b4263 100644 --- a/wolnelektury/settings/basic.py +++ b/wolnelektury/settings/basic.py @@ -22,7 +22,7 @@ DATABASES = { } SOLR = "http://localhost:8983/solr/wl/" - +SOLR_TEST = "http://localhost:8983/solr/wl_test/" # Local time zone for this installation. Choices can be found here: # http://en.wikipedia.org/wiki/List_of_tz_zones_by_name