X-Git-Url: https://git.mdrn.pl/wolnelektury.git/blobdiff_plain/f59e7c3de6bd0f85a61a4d9481db60cd7369ae92..1e9113af163a3d002b0b9c3e8c339f22af5b848a:/apps/opds/views.py diff --git a/apps/opds/views.py b/apps/opds/views.py index c907fe198..23c320e62 100644 --- a/apps/opds/views.py +++ b/apps/opds/views.py @@ -2,10 +2,8 @@ # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later. # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information. # -from base64 import b64encode import os.path from urlparse import urljoin -from urllib2 import unquote from django.contrib.syndication.views import Feed from django.core.urlresolvers import reverse @@ -18,11 +16,13 @@ from django.contrib.sites.models import Site from basicauth import logged_in_or_basicauth, factory_decorator from catalogue.models import Book, Tag -from search import MultiSearch, SearchResult, JVM -from lucene import Term, QueryWrapperFilter, TermQuery - +from search.views import Search, SearchResult +import operator +import logging import re +log = logging.getLogger('opds') + from stats.utils import piwik_track _root_feeds = ( @@ -91,7 +91,7 @@ class OPDSFeed(Atom1Feed): {u"href": reverse("opds_authors"), u"rel": u"start", u"type": u"application/atom+xml"}) - handler.addQuickElement(u"link", None, + handler.addQuickElement(u"link", None, {u"href": full_url(os.path.join(settings.STATIC_URL, "opensearch.xml")), u"rel": u"search", u"type": u"application/opensearchdescription+xml"}) @@ -187,19 +187,19 @@ class AcquisitionFeed(Feed): return u'' def item_enclosure_url(self, book): - return full_url(book.root_ancestor.epub_file.url) + return full_url(book.epub_file.url) if book.epub_file else None def item_enclosure_length(self, book): - return book.root_ancestor.epub_file.size + return book.epub_file.size if book.epub_file else None @piwik_track class RootFeed(Feed): feed_type = OPDSFeed title = u'Wolne Lektury' - link = u'http://www.wolnelektury.pl/' + link = u'http://wolnelektury.pl/' description = u"Spis utworów na stronie http://WolneLektury.pl" author_name = u"Wolne Lektury" - author_link = u"http://www.wolnelektury.pl/" + author_link = u"http://wolnelektury.pl/" def items(self): return _root_feeds @@ -216,10 +216,10 @@ class RootFeed(Feed): @piwik_track class ByCategoryFeed(Feed): feed_type = OPDSFeed - link = u'http://www.wolnelektury.pl/' + link = u'http://wolnelektury.pl/' description = u"Spis utworów na stronie http://WolneLektury.pl" author_name = u"Wolne Lektury" - author_link = u"http://www.wolnelektury.pl/" + author_link = u"http://wolnelektury.pl/" def get_object(self, request, category): feed = [feed for feed in _root_feeds if feed['category']==category] @@ -261,7 +261,7 @@ class ByTagFeed(AcquisitionFeed): def items(self, tag): books = Book.tagged.with_any([tag]) - l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in books]) + l_tags = Tag.objects.filter(category='book', slug__in=[book.book_tag_slug() for book in books.iterator()]) descendants_keys = [book.pk for book in Book.tagged.with_any(l_tags)] if descendants_keys: books = books.exclude(pk__in=descendants_keys) @@ -276,7 +276,7 @@ class UserFeed(Feed): link = u'http://www.wolnelektury.pl/' description = u"Półki użytkownika na stronie http://WolneLektury.pl" author_name = u"Wolne Lektury" - author_link = u"http://www.wolnelektury.pl/" + author_link = u"http://wolnelektury.pl/" def get_object(self, request): return request.user @@ -327,8 +327,35 @@ class SearchFeed(AcquisitionFeed): description = u"Wyniki wyszukiwania na stronie WolneLektury.pl" title = u"Wyniki wyszukiwania" - INLINE_QUERY_RE = re.compile(r"(author:(?P[^ ]+)|title:(?P[^ ]+)|categories:(?P<categories>[^ ]+)|description:(?P<description>[^ ]+))") - + QUOTE_OR_NOT = r'(?:(?=["])"([^"]+)"|([^ ]+))' + INLINE_QUERY_RE = re.compile( + r"author:" + QUOTE_OR_NOT + + "|translator:" + QUOTE_OR_NOT + + "|title:" + QUOTE_OR_NOT + + "|categories:" + QUOTE_OR_NOT + + "|description:" + QUOTE_OR_NOT + + "|text:" + QUOTE_OR_NOT + ) + MATCHES = { + 'author': (0, 1), + 'translator': (2, 3), + 'title': (4, 5), + 'categories': (6, 7), + 'description': (8, 9), + 'text': (10, 11), + } + + PARAMS_TO_FIELDS = { + 'author': 'authors', + 'translator': 'translators', + # 'title': 'title', + 'categories': 'tag_name_pl', + 'description': 'text', + # 'text': 'text', + } + + ATOM_PLACEHOLDER = re.compile(r"^{(atom|opds):\w+}$") + def get_object(self, request): """ For OPDS 1.1 We should handle a query for search terms @@ -336,7 +363,7 @@ class SearchFeed(AcquisitionFeed): OpenSearch defines fields: atom:author, atom:contributor (treated as translator), atom:title. Inline query provides author, title, categories (treated as book tags), description (treated as content search terms). - + if search terms are provided, we shall search for books according to Hint information (from author & contributror & title). @@ -344,94 +371,67 @@ class SearchFeed(AcquisitionFeed): (perhaps for is_book=True) """ - JVM.attachCurrentThread() query = request.GET.get('q', '') inline_criteria = re.findall(self.INLINE_QUERY_RE, query) if inline_criteria: - def get_criteria(criteria, name, position): - e = filter(lambda el: el[0][0:len(name)] == name, criteria) - print e - if not e: - return None - c = e[0][position] - print c - if c[0] == '"' and c[-1] == '"': - c = c[1:-1] - c = c.replace('+', ' ') - return c - - #import pdb; pdb.set_trace() - author = get_criteria(inline_criteria, 'author', 1) - title = get_criteria(inline_criteria, 'title', 2) - translator = None - categories = get_criteria(inline_criteria, 'categories', 3) - query = get_criteria(inline_criteria, 'description', 4) + remains = re.sub(self.INLINE_QUERY_RE, '', query) + remains = re.sub(r'[ \t]+', ' ', remains) + + def get_criteria(criteria, name): + for c in criteria: + for p in self.MATCHES[name]: + if c[p]: + if p % 2 == 0: + return c[p].replace('+', ' ') + return c[p] + return None + + criteria = dict(map( + lambda cn: (cn, get_criteria(inline_criteria, cn)), + ['author', 'translator', 'title', 'categories', + 'description', 'text'])) + query = remains + # empty query and text set case? + log.debug("Inline query = [%s], criteria: %s" % (query, criteria)) else: - author = request.GET.get('author', '') - title = request.GET.get('title', '') - translator = request.GET.get('translator', '') - categories = None - fuzzy = False + def remove_dump_data(val): + """Some clients don't get opds placeholders and just send them.""" + if self.ATOM_PLACEHOLDER.match(val): + return '' + return val + criteria = dict([(cn, remove_dump_data(request.GET.get(cn, ''))) + for cn in self.MATCHES.keys()]) + # query is set above. + log.debug("Inline query = [%s], criteria: %s" % (query, criteria)) - srch = MultiSearch() - hint = srch.hint() + srch = Search() + + book_hit_filter = srch.index.Q(book_id__any=True) + filters = [book_hit_filter] + [srch.index.Q( + **{self.PARAMS_TO_FIELDS.get(cn, cn): criteria[cn]} + ) for cn in self.MATCHES.keys() if cn in criteria + if criteria[cn]] - # Scenario 1: full search terms provided. - # Use auxiliarry information to narrow it and make it better. if query: - filters = [] - - if author: - print "narrow to author %s" % author - hint.tags(srch.search_tags(author, filter=srch.term_filter(Term('tag_category', 'author')))) - - if translator: - print "filter by translator %s" % translator - filters.append(QueryWrapperFilter( - srch.make_phrase(srch.get_tokens(translator, field='translators'), - field='translators'))) - - if categories: - filters.append(QueryWrapperFilter( - srch.make_phrase(srch.get_tokens(categories, field="tag_name_pl"), - field='tag_name_pl'))) - - flt = srch.chain_filters(filters) - if title: - print "hint by book title %s" % title - q = srch.make_phrase(srch.get_tokens(title, field='title'), field='title') - hint.books(*srch.search_books(q, filter=flt)) - - toks = srch.get_tokens(query) - print "tokens: %s" % toks - # import pdb; pdb.set_trace() - results = SearchResult.aggregate(srch.search_perfect_book(toks, fuzzy=fuzzy, hint=hint), - srch.search_perfect_parts(toks, fuzzy=fuzzy, hint=hint), - srch.search_everywhere(toks, fuzzy=fuzzy, hint=hint)) - results.sort(reverse=True) - return [r.book for r in results] + q = srch.index.query( + reduce(operator.or_, + [srch.index.Q(**{self.PARAMS_TO_FIELDS.get(cn, cn): query}) + for cn in self.MATCHES.keys()], + srch.index.Q())) else: - # Scenario 2: since we no longer have to figure out what the query term means to the user, - # we can just use filters and not the Hint class. - filters = [] - - fields = { - 'author': author, - 'translators': translator, - 'title': title - } - - for fld, q in fields.items(): - if q: - filters.append(QueryWrapperFilter( - srch.make_phrase(srch.get_tokens(q, field=fld), field=fld))) - - flt = srch.chain_filters(filters) - books = srch.search_books(TermQuery(Term('is_book', 'true')), filter=flt) - return books + q = srch.index.query(srch.index.Q()) + + q = srch.apply_filters(q, filters).field_limit(score=True, fields=['book_id']) + results = q.execute() + + book_scores = dict([(r['book_id'], r['score']) for r in results]) + books = Book.objects.filter(id__in=set([r['book_id'] for r in results])) + books = list(books) + books.sort(reverse=True, key=lambda book: book_scores[book.id]) + return books def get_link(self, query): return "%s?q=%s" % (reverse('search'), query)