X-Git-Url: https://git.mdrn.pl/wolnelektury.git/blobdiff_plain/18a5524b295619382c9438b8de44627e3acc28b0..cf287e8434ef1d5e84be9f6a3047f73eb0721285:/src/search/index.py?ds=sidebyside diff --git a/src/search/index.py b/src/search/index.py index c84841b19..ab3286aee 100644 --- a/src/search/index.py +++ b/src/search/index.py @@ -20,6 +20,13 @@ from wolnelektury.utils import makedirs log = logging.getLogger('search') +if os.path.isfile(settings.SOLR_STOPWORDS): + stopwords = set( + line.decode('utf-8').strip() + for line in open(settings.SOLR_STOPWORDS) if not line.startswith('#')) +else: + stopwords = set() + class SolrIndex(object): def __init__(self, mode=None): @@ -240,7 +247,8 @@ class Index(SolrIndex): self.remove_book(book, remove_snippets=False) book_doc = self.create_book_doc(book) - meta_fields = self.extract_metadata(book, book_info, dc_only=['source_name', 'authors', 'translators', 'title']) + meta_fields = self.extract_metadata(book, book_info, dc_only=[ + 'source_name', 'authors', 'translators', 'title', 'epochs', 'kinds', 'genres']) # let's not index it - it's only used for extracting publish date if 'source_name' in meta_fields: del meta_fields['source_name'] @@ -257,8 +265,9 @@ class Index(SolrIndex): 'published_date': meta_fields['published_date'] } - if 'translators' in meta_fields: - book_fields['translators'] = meta_fields['translators'] + for tag_name in ('translators', 'epochs', 'kinds', 'genres'): + if tag_name in meta_fields: + book_fields[tag_name] = meta_fields[tag_name] self.index_content(book, book_fields=book_fields) @@ -269,14 +278,14 @@ class Index(SolrIndex): 'dramat_wierszowany_lp', 'dramat_wspolczesny', 'liryka_l', 'liryka_lp', 'wywiad', - ] + ] ignore_content_tags = [ - 'uwaga', 'extra', + 'uwaga', 'extra', 'nota_red', 'abstrakt', 'zastepnik_tekstu', 'sekcja_asterysk', 'separator_linia', 'zastepnik_wersu', 'didaskalia', 'naglowek_aktu', 'naglowek_sceny', 'naglowek_czesc', - ] + ] footnote_tags = ['pa', 'pt', 'pr', 'pe'] @@ -365,8 +374,8 @@ class Index(SolrIndex): if master is None: return [] - def walker(node, ignore_tags=()): - if node.tag not in ignore_tags: + def walker(node): + if node.tag not in self.ignore_content_tags: yield node, None, None if node.text is not None: yield None, node.text, None @@ -417,17 +426,10 @@ class Index(SolrIndex): if 'themes' in fields: doc['themes'] = fields['themes'] - doc['uid'] = "part%s%s%s" % (doc['header_index'], - doc['header_span'], - doc.get('fragment_anchor', '')) + doc['uid'] = "part%s-%s-%s-%s" % ( + book.id, doc['header_index'], doc['header_span'], doc.get('fragment_anchor', '')) return doc - def give_me_utf8(s): - if isinstance(s, unicode): - return s.encode('utf-8') - else: - return s - fragments = {} snippets = Snippets(book.id).open('w') try: @@ -448,7 +450,7 @@ class Index(SolrIndex): content.append(text) handle_text = [all_content] - for start, text, end in walker(header, ignore_tags=self.ignore_content_tags): + for start, text, end in walker(header): # handle footnotes if start is not None and start.tag in self.footnote_tags: footnote = [] @@ -515,8 +517,7 @@ class Index(SolrIndex): class SearchResult(object): - def __init__(self, doc, how_found=None, query=None, query_terms=None): - # self.search = search + def __init__(self, doc, how_found=None, query_terms=None): self.boost = 1.0 self._hits = [] self._processed_hits = None # processed hits @@ -572,10 +573,9 @@ class SearchResult(object): def merge(self, other): if self.book_id != other.book_id: - raise ValueError("this search result is or book %d; tried to merge with %d" % (self.book_id, other.book_id)) + raise ValueError("this search result is for book %d; tried to merge with %d" % (self.book_id, other.book_id)) self._hits += other._hits - if other.score > self.score: - self._score = other._score + self._score += max(other._score, 0) + 0.5 return self def get_book(self): @@ -676,7 +676,7 @@ class SearchResult(object): m.update(f[self.OTHER]) hits.append(m) - hits.sort(lambda a, b: cmp(a['score'], b['score']), reverse=True) + hits.sort(key=lambda h: h['score'], reverse=True) self._processed_hits = hits @@ -734,71 +734,26 @@ class Search(SolrIndex): return q - def search_phrase(self, searched, field='text', book=False, - filters=None, - snippets=False): - if filters is None: - filters = [] - if book: - filters.append(self.index.Q(is_book=True)) - - q = self.index.query(**{field: searched}) - q = self.apply_filters(q, filters).field_limit(score=True, all_fields=True) - res = q.execute() - return [SearchResult(found, how_found=u'search_phrase') for found in res] - - def search_some(self, searched, fields, book=True, - filters=None, snippets=True, query_terms=None): - assert isinstance(fields, list) - if filters is None: - filters = [] + def search_words(self, words, fields, book=True): + filters = [] + for word in words: + if word not in stopwords: + word_filter = None + for field in fields: + q = self.index.Q(**{field: word}) + if word_filter is None: + word_filter = q + else: + word_filter |= q + filters.append(word_filter) + if not filters: + return [] if book: - filters.append(self.index.Q(is_book=True)) - - query = self.index.Q() - - for fld in fields: - query = self.index.Q(query | self.make_term_query(searched, fld)) - - query = self.index.query(query) + query = self.index.query(is_book=True) + else: + query = self.index.query() query = self.apply_filters(query, filters).field_limit(score=True, all_fields=True) - res = query.execute() - return [SearchResult(found, how_found='search_some', query_terms=query_terms) for found in res] - - def search_everywhere(self, searched, query_terms=None): - """ - Tries to use search terms to match different fields of book (or its parts). - E.g. one word can be an author survey, another be a part of the title, and the rest - are some words from third chapter. - """ - books = [] - # content only query : themes x content - q = self.make_term_query(searched, 'text') - q_themes = self.make_term_query(searched, 'themes_pl') - - query = self.index.query(q).query(q_themes).field_limit(score=True, all_fields=True) - res = query.execute() - - for found in res: - books.append(SearchResult(found, how_found='search_everywhere_themesXcontent', query_terms=query_terms)) - - # query themes/content x author/title/tags - in_content = self.index.Q() - in_meta = self.index.Q() - - for fld in ['themes_pl', 'text']: - in_content |= self.make_term_query(searched, field=fld) - - for fld in ['tags', 'authors', 'title']: - in_meta |= self.make_term_query(searched, field=fld) - - q = in_content & in_meta - res = self.index.query(q).field_limit(score=True, all_fields=True).execute() - - for found in res: - books.append(SearchResult(found, how_found='search_everywhere', query_terms=query_terms)) - - return books + return [SearchResult(found, how_found='search_words', query_terms=words) for found in query.execute()] def get_snippets(self, searchresult, query, field='text', num=1): """ @@ -821,14 +776,17 @@ class Search(SolrIndex): text = snippets.get((int(position), int(length))) snip = self.index.highlight(text=text, field=field, q=query) - snips[idx] = snip - if snip: - num -= 1 + if snip not in snips: + snips[idx] = snip + if snip: + num -= 1 idx += 1 except IOError, e: - book = catalogue.models.Book.objects.get(id=book_id) - if not book.children.exists(): + book = catalogue.models.Book.objects.filter(id=book_id) + if not book: + log.error("Book does not exist for book id = %d" % book_id) + elif not book.get().children.exists(): log.error("Cannot open snippet file for book id = %d [rev=%s], %s" % (book_id, revision, e)) return [] finally: @@ -841,102 +799,6 @@ class Search(SolrIndex): return snips - def hint_tags(self, query, pdcounter=True, prefix=True): - """ - Return auto-complete hints for tags - using prefix search. - """ - q = self.index.Q() - query = query.strip() - for field in ['tag_name', 'tag_name_pl']: - if prefix: - q |= self.index.Q(**{field: query + "*"}) - else: - q |= self.make_term_query(query, field=field) - qu = self.index.query(q) - - return self.search_tags(qu, pdcounter=pdcounter) - - def search_tags(self, query, filters=None, pdcounter=False): - """ - Search for Tag objects using query. - """ - if not filters: - filters = [] - if not pdcounter: - filters.append(~self.index.Q(is_pdcounter=True)) - res = self.apply_filters(query, filters).execute() - - tags = [] - pd_tags = [] - - for doc in res: - is_pdcounter = doc.get('is_pdcounter', False) - category = doc.get('tag_category') - try: - if is_pdcounter: - if category == 'pd_author': - tag = PDCounterAuthor.objects.get(id=doc.get('tag_id')) - elif category == 'pd_book': - tag = PDCounterBook.objects.get(id=doc.get('tag_id')) - tag.category = 'pd_book' # make it look more lik a tag. - else: - # WTF - print ("Warning. cannot get pdcounter tag_id=%d from db; cat=%s" % ( - int(doc.get('tag_id')), category)).encode('utf-8') - pd_tags.append(tag) - else: - tag = catalogue.models.Tag.objects.get(id=doc.get("tag_id")) - tags.append(tag) - - except catalogue.models.Tag.DoesNotExist: - pass - except PDCounterAuthor.DoesNotExist: - pass - except PDCounterBook.DoesNotExist: - pass - - tags_slugs = set(map(lambda t: t.slug, tags)) - tags = tags + filter(lambda t: t.slug not in tags_slugs, pd_tags) - - log.debug('search_tags: %s' % tags) - - return tags - - def hint_books(self, query, prefix=True): - """ - Returns auto-complete hints for book titles - Because we do not index 'pseudo' title-tags. - Prefix search. - """ - q = self.index.Q() - query = query.strip() - if prefix: - q |= self.index.Q(title=query + "*") - else: - q |= self.make_term_query(query, field='title') - qu = self.index.query(q) - only_books = self.index.Q(is_book=True) - return self.search_books(qu, [only_books]) - - def search_books(self, query, filters=None, max_results=10): - """ - Searches for Book objects using query - """ - bks = [] - bks_found = set() - query = query.query(is_book=True) - res = self.apply_filters(query, filters).field_limit(['book_id']) - for r in res: - try: - bid = r['book_id'] - if bid not in bks_found: - bks.append(catalogue.models.Book.objects.get(id=bid)) - bks_found.add(bid) - except catalogue.models.Book.DoesNotExist: - pass - return bks - @staticmethod def apply_filters(query, filters): """