fix indexing bug (conflicting uids)
[wolnelektury.git] / src / search / index.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from django.conf import settings
6
7 import os
8 import re
9 from librarian import dcparser
10 from librarian.parser import WLDocument
11 from lxml import etree
12 import catalogue.models
13 from pdcounter.models import Author as PDCounterAuthor, BookStub as PDCounterBook
14 from itertools import chain
15 import sunburnt
16 import custom
17 import operator
18 import logging
19 from wolnelektury.utils import makedirs
20
21 log = logging.getLogger('search')
22
23
24 class SolrIndex(object):
25     def __init__(self, mode=None):
26         self.index = custom.CustomSolrInterface(settings.SOLR, mode=mode)
27
28
29 class Snippets(object):
30     """
31     This class manages snippet files for indexed object (book)
32     the snippets are concatenated together, and their positions and
33     lengths are kept in lucene index fields.
34     """
35     SNIPPET_DIR = "snippets"
36
37     def __init__(self, book_id, revision=None):
38         makedirs(os.path.join(settings.SEARCH_INDEX, self.SNIPPET_DIR))
39         self.book_id = book_id
40         self.revision = revision
41         self.file = None
42         self.position = None
43
44     @property
45     def path(self):
46         if self.revision:
47             fn = "%d.%d" % (self.book_id, self.revision)
48         else:
49             fn = "%d" % self.book_id
50
51         return os.path.join(settings.SEARCH_INDEX, self.SNIPPET_DIR, fn)
52
53     def open(self, mode='r'):
54         """
55         Open the snippet file. Call .close() afterwards.
56         """
57         if 'b' not in mode:
58             mode += 'b'
59
60         if 'w' in mode:
61             if os.path.exists(self.path):
62                 self.revision = 1
63                 while True:
64                     if not os.path.exists(self.path):
65                         break
66                     self.revision += 1
67
68         self.file = open(self.path, mode)
69         self.position = 0
70         return self
71
72     def add(self, snippet):
73         """
74         Append a snippet (unicode) to the snippet file.
75         Return a (position, length) tuple
76         """
77         txt = snippet.encode('utf-8')
78         l = len(txt)
79         self.file.write(txt)
80         pos = (self.position, l)
81         self.position += l
82         return pos
83
84     def get(self, pos):
85         """
86         Given a tuple of (position, length) return an unicode
87         of the snippet stored there.
88         """
89         self.file.seek(pos[0], 0)
90         txt = self.file.read(pos[1]).decode('utf-8')
91         return txt
92
93     def close(self):
94         """Close snippet file"""
95         if self.file:
96             self.file.close()
97
98     def remove(self):
99         self.revision = None
100         try:
101             os.unlink(self.path)
102             self.revision = 0
103             while True:
104                 self.revision += 1
105                 os.unlink(self.path)
106         except OSError:
107             pass
108
109
110 class Index(SolrIndex):
111     """
112     Class indexing books.
113     """
114     def __init__(self):
115         super(Index, self).__init__(mode='rw')
116
117     def delete_query(self, *queries):
118         """
119         index.delete(queries=...) doesn't work, so let's reimplement it
120         using deletion of list of uids.
121         """
122         uids = set()
123         for q in queries:
124             if isinstance(q, sunburnt.search.LuceneQuery):
125                 q = self.index.query(q)
126             q.field_limiter.update(['uid'])
127             st = 0
128             rows = 100
129             while True:
130                 ids = q.paginate(start=st, rows=rows).execute()
131                 if not len(ids):
132                     break
133                 for res in ids:
134                     uids.add(res['uid'])
135                 st += rows
136         if uids:
137             self.index.delete(uids)
138             return True
139         else:
140             return False
141
142     def index_tags(self, *tags, **kw):
143         """
144         Re-index global tag list.
145         Removes all tags from index, then index them again.
146         Indexed fields include: id, name (with and without polish stems), category
147         """
148         log.debug("Indexing tags")
149         remove_only = kw.get('remove_only', False)
150         # first, remove tags from index.
151         if tags:
152             tag_qs = []
153             for tag in tags:
154                 q_id = self.index.Q(tag_id=tag.id)
155
156                 if isinstance(tag, PDCounterAuthor):
157                     q_cat = self.index.Q(tag_category='pd_author')
158                 elif isinstance(tag, PDCounterBook):
159                     q_cat = self.index.Q(tag_category='pd_book')
160                 else:
161                     q_cat = self.index.Q(tag_category=tag.category)
162
163                 q_id_cat = self.index.Q(q_id & q_cat)
164                 tag_qs.append(q_id_cat)
165             self.delete_query(*tag_qs)
166         else:  # all
167             q = self.index.Q(tag_id__any=True)
168             self.delete_query(q)
169
170         if not remove_only:
171             # then add them [all or just one passed]
172             if not tags:
173                 tags = chain(
174                     catalogue.models.Tag.objects.exclude(category='set'),
175                     PDCounterAuthor.objects.all(),
176                     PDCounterBook.objects.all())
177
178             for tag in tags:
179                 if isinstance(tag, PDCounterAuthor):
180                     doc = {
181                         "tag_id": int(tag.id),
182                         "tag_name": tag.name,
183                         "tag_name_pl": tag.name,
184                         "tag_category": 'pd_author',
185                         "is_pdcounter": True,
186                         "uid": "tag%d_pd_a" % tag.id
187                         }
188                 elif isinstance(tag, PDCounterBook):
189                     doc = {
190                         "tag_id": int(tag.id),
191                         "tag_name": tag.title,
192                         "tag_name_pl": tag.title,
193                         "tag_category": 'pd_book',
194                         "is_pdcounter": True,
195                         "uid": "tag%d_pd_b" % tag.id
196                         }
197                 else:
198                     doc = {
199                         "tag_id": int(tag.id),
200                         "tag_name": tag.name,
201                         "tag_name_pl": tag.name,
202                         "tag_category": tag.category,
203                         "is_pdcounter": False,
204                         "uid": "tag%d" % tag.id
205                         }
206                 self.index.add(doc)
207
208     def create_book_doc(self, book):
209         """
210         Create a lucene document referring book id.
211         """
212         doc = {'book_id': int(book.id)}
213         if book.parent is not None:
214             doc['parent_id'] = int(book.parent.id)
215         return doc
216
217     def remove_book(self, book_or_id, remove_snippets=True):
218         """Removes a book from search index.
219         book - Book instance."""
220         if isinstance(book_or_id, catalogue.models.Book):
221             book_id = book_or_id.id
222         else:
223             book_id = book_or_id
224
225         self.delete_query(self.index.Q(book_id=book_id))
226
227         if remove_snippets:
228             snippets = Snippets(book_id)
229             snippets.remove()
230
231     def index_book(self, book, book_info=None, overwrite=True):
232         """
233         Indexes the book.
234         Creates a lucene document for extracted metadata
235         and calls self.index_content() to index the contents of the book.
236         """
237         if overwrite:
238             # we don't remove snippets, since they might be still needed by
239             # threads using not reopened index
240             self.remove_book(book, remove_snippets=False)
241
242         book_doc = self.create_book_doc(book)
243         meta_fields = self.extract_metadata(book, book_info, dc_only=['source_name', 'authors', 'translators', 'title'])
244         # let's not index it - it's only used for extracting publish date
245         if 'source_name' in meta_fields:
246             del meta_fields['source_name']
247
248         for n, f in meta_fields.items():
249             book_doc[n] = f
250
251         book_doc['uid'] = "book%s" % book_doc['book_id']
252         self.index.add(book_doc)
253         del book_doc
254         book_fields = {
255             'title': meta_fields['title'],
256             'authors': meta_fields['authors'],
257             'published_date': meta_fields['published_date']
258             }
259
260         if 'translators' in meta_fields:
261             book_fields['translators'] = meta_fields['translators']
262
263         self.index_content(book, book_fields=book_fields)
264
265     master_tags = [
266         'opowiadanie',
267         'powiesc',
268         'dramat_wierszowany_l',
269         'dramat_wierszowany_lp',
270         'dramat_wspolczesny', 'liryka_l', 'liryka_lp',
271         'wywiad',
272         ]
273
274     ignore_content_tags = [
275         'uwaga', 'extra',
276         'zastepnik_tekstu', 'sekcja_asterysk', 'separator_linia', 'zastepnik_wersu',
277         'didaskalia',
278         'naglowek_aktu', 'naglowek_sceny', 'naglowek_czesc',
279         ]
280
281     footnote_tags = ['pa', 'pt', 'pr', 'pe']
282
283     skip_header_tags = ['autor_utworu', 'nazwa_utworu', 'dzielo_nadrzedne',
284                         '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF']
285
286     published_date_re = re.compile("([0-9]+)[\]. ]*$")
287
288     def extract_metadata(self, book, book_info=None, dc_only=None):
289         """
290         Extract metadata from book and returns a map of fields keyed by fieldname
291         """
292         fields = {}
293
294         if book_info is None:
295             book_info = dcparser.parse(open(book.xml_file.path))
296
297         fields['slug'] = book.slug
298         fields['tags'] = [t.name for t in book.tags]
299         fields['is_book'] = True
300
301         # validator, name
302         for field in dcparser.BookInfo.FIELDS:
303             if dc_only and field.name not in dc_only:
304                 continue
305             if hasattr(book_info, field.name):
306                 if not getattr(book_info, field.name):
307                     continue
308                 # since no type information is available, we use validator
309                 type_indicator = field.validator
310                 if type_indicator == dcparser.as_unicode:
311                     s = getattr(book_info, field.name)
312                     if field.multiple:
313                         s = ', '.join(s)
314                     fields[field.name] = s
315                 elif type_indicator == dcparser.as_person:
316                     p = getattr(book_info, field.name)
317                     if isinstance(p, dcparser.Person):
318                         persons = unicode(p)
319                     else:
320                         persons = ', '.join(map(unicode, p))
321                     fields[field.name] = persons
322                 elif type_indicator == dcparser.as_date:
323                     dt = getattr(book_info, field.name)
324                     fields[field.name] = dt
325
326         # get published date
327         pd = None
328         if hasattr(book_info, 'source_name') and book_info.source_name:
329             match = self.published_date_re.search(book_info.source_name)
330             if match is not None:
331                 pd = str(match.groups()[0])
332         if not pd:
333             pd = ""
334         fields["published_date"] = pd
335
336         return fields
337
338     # def add_gaps(self, fields, fieldname):
339     #     """
340     #     Interposes a list of fields with gap-fields, which are indexed spaces and returns it.
341     #     This allows for doing phrase queries which do not overlap the gaps (when slop is 0).
342     #     """
343     #     def gap():
344     #         while True:
345     #             yield Field(fieldname, ' ', Field.Store.NO, Field.Index.NOT_ANALYZED)
346     #     return reduce(lambda a, b: a + b, zip(fields, gap()))[0:-1]
347
348     def get_master(self, root):
349         """
350         Returns the first master tag from an etree.
351         """
352         for master in root.iter():
353             if master.tag in self.master_tags:
354                 return master
355
356     def index_content(self, book, book_fields):
357         """
358         Walks the book XML and extract content from it.
359         Adds parts for each header tag and for each fragment.
360         """
361         wld = WLDocument.from_file(book.xml_file.path, parse_dublincore=False)
362         root = wld.edoc.getroot()
363
364         master = self.get_master(root)
365         if master is None:
366             return []
367
368         def walker(node, ignore_tags=()):
369             if node.tag not in ignore_tags:
370                 yield node, None, None
371                 if node.text is not None:
372                     yield None, node.text, None
373                 for child in list(node):
374                     for b, t, e in walker(child):
375                         yield b, t, e
376                 yield None, None, node
377
378             if node.tail is not None:
379                 yield None, node.tail, None
380             return
381
382         def fix_format(text):
383             # separator = [u" ", u"\t", u".", u";", u","]
384             if isinstance(text, list):
385                 # need to join it first
386                 text = filter(lambda s: s is not None, content)
387                 text = u' '.join(text)
388                 # for i in range(len(text)):
389                 #     if i > 0:
390                 #         if text[i][0] not in separator\
391                 #             and text[i - 1][-1] not in separator:
392                 #          text.insert(i, u" ")
393
394             return re.sub("(?m)/$", "", text)
395
396         def add_part(snippets, **fields):
397             doc = self.create_book_doc(book)
398             for n, v in book_fields.items():
399                 doc[n] = v
400
401             doc['header_index'] = fields["header_index"]
402             doc['header_span'] = 'header_span' in fields and fields['header_span'] or 1
403             doc['header_type'] = fields['header_type']
404
405             doc['text'] = fields['text']
406
407             # snippets
408             snip_pos = snippets.add(fields["text"])
409
410             doc['snippets_position'] = snip_pos[0]
411             doc['snippets_length'] = snip_pos[1]
412             if snippets.revision:
413                 doc["snippets_revision"] = snippets.revision
414
415             if 'fragment_anchor' in fields:
416                 doc["fragment_anchor"] = fields['fragment_anchor']
417
418             if 'themes' in fields:
419                 doc['themes'] = fields['themes']
420             doc['uid'] = "part%s-%s-%s-%s" % (
421                 book.id, doc['header_index'], doc['header_span'], doc.get('fragment_anchor', ''))
422             return doc
423
424         def give_me_utf8(s):
425             if isinstance(s, unicode):
426                 return s.encode('utf-8')
427             else:
428                 return s
429
430         fragments = {}
431         snippets = Snippets(book.id).open('w')
432         try:
433             for header, position in zip(list(master), range(len(master))):
434
435                 if header.tag in self.skip_header_tags:
436                     continue
437                 if header.tag is etree.Comment:
438                     continue
439
440                 # section content
441                 content = []
442                 footnote = []
443
444                 def all_content(text):
445                     for frag in fragments.values():
446                         frag['text'].append(text)
447                     content.append(text)
448                 handle_text = [all_content]
449
450                 for start, text, end in walker(header, ignore_tags=self.ignore_content_tags):
451                     # handle footnotes
452                     if start is not None and start.tag in self.footnote_tags:
453                         footnote = []
454
455                         def collect_footnote(t):
456                             footnote.append(t)
457
458                         handle_text.append(collect_footnote)
459                     elif end is not None and footnote is not [] and end.tag in self.footnote_tags:
460                         handle_text.pop()
461                         doc = add_part(snippets, header_index=position, header_type=header.tag,
462                                        text=u''.join(footnote),
463                                        is_footnote=True)
464                         self.index.add(doc)
465                         footnote = []
466
467                     # handle fragments and themes.
468                     if start is not None and start.tag == 'begin':
469                         fid = start.attrib['id'][1:]
470                         fragments[fid] = {
471                             'text': [], 'themes': [], 'start_section': position, 'start_header': header.tag}
472
473                     # themes for this fragment
474                     elif start is not None and start.tag == 'motyw':
475                         fid = start.attrib['id'][1:]
476                         handle_text.append(lambda text: None)
477                         if start.text is not None:
478                             fragments[fid]['themes'] += map(unicode.strip, map(unicode, (start.text.split(','))))
479                     elif end is not None and end.tag == 'motyw':
480                         handle_text.pop()
481
482                     elif start is not None and start.tag == 'end':
483                         fid = start.attrib['id'][1:]
484                         if fid not in fragments:
485                             continue  # a broken <end> node, skip it
486                         frag = fragments[fid]
487                         if not frag['themes']:
488                             continue  # empty themes list.
489                         del fragments[fid]
490
491                         doc = add_part(snippets,
492                                        header_type=frag['start_header'],
493                                        header_index=frag['start_section'],
494                                        header_span=position - frag['start_section'] + 1,
495                                        fragment_anchor=fid,
496                                        text=fix_format(frag['text']),
497                                        themes=frag['themes'])
498                         self.index.add(doc)
499
500                         # Collect content.
501
502                     if text is not None and handle_text is not []:
503                         hdl = handle_text[-1]
504                         hdl(text)
505
506                         # in the end, add a section text.
507                 doc = add_part(snippets, header_index=position,
508                                header_type=header.tag, text=fix_format(content))
509
510                 self.index.add(doc)
511
512         finally:
513             snippets.close()
514
515
516 class SearchResult(object):
517     def __init__(self, doc, how_found=None, query_terms=None):
518         self.boost = 1.0
519         self._hits = []
520         self._processed_hits = None  # processed hits
521         self.snippets = []
522         self.query_terms = query_terms
523         self._book = None
524
525         if 'score' in doc:
526             self._score = doc['score']
527         else:
528             self._score = 0
529
530         self.book_id = int(doc["book_id"])
531
532         try:
533             self.published_date = int(doc.get("published_date"))
534         except ValueError:
535             self.published_date = 0
536
537         # content hits
538         header_type = doc.get("header_type", None)
539         # we have a content hit in some header of fragment
540         if header_type is not None:
541             sec = (header_type, int(doc["header_index"]))
542             header_span = doc['header_span']
543             header_span = header_span is not None and int(header_span) or 1
544             fragment = doc.get("fragment_anchor", None)
545             snippets_pos = (doc['snippets_position'], doc['snippets_length'])
546             snippets_rev = doc.get('snippets_revision', None)
547
548             hit = (sec + (header_span,), fragment, self._score, {
549                 'how_found': how_found,
550                 'snippets_pos': snippets_pos,
551                 'snippets_revision': snippets_rev,
552                 'themes': doc.get('themes', []),
553                 'themes_pl': doc.get('themes_pl', [])
554                 })
555
556             self._hits.append(hit)
557
558     def __unicode__(self):
559         return u"<SR id=%d %d(%d) hits score=%f %d snippets>" % \
560             (self.book_id, len(self._hits),
561              len(self._processed_hits) if self._processed_hits else -1,
562              self._score, len(self.snippets))
563
564     def __str__(self):
565         return unicode(self).encode('utf-8')
566
567     @property
568     def score(self):
569         return self._score * self.boost
570
571     def merge(self, other):
572         if self.book_id != other.book_id:
573             raise ValueError("this search result is or book %d; tried to merge with %d" % (self.book_id, other.book_id))
574         self._hits += other._hits
575         if other.score > self.score:
576             self._score = other._score
577         return self
578
579     def get_book(self):
580         if self._book is not None:
581             return self._book
582         self._book = catalogue.models.Book.objects.get(id=self.book_id)
583         return self._book
584
585     book = property(get_book)
586
587     POSITION = 0
588     FRAGMENT = 1
589     POSITION_INDEX = 1
590     POSITION_SPAN = 2
591     SCORE = 2
592     OTHER = 3
593
594     @property
595     def hits(self):
596         if self._processed_hits is not None:
597             return self._processed_hits
598
599         # to sections and fragments
600         frags = filter(lambda r: r[self.FRAGMENT] is not None, self._hits)
601
602         sect = filter(lambda r: r[self.FRAGMENT] is None, self._hits)
603
604         # sections not covered by fragments
605         sect = filter(lambda s: 0 == len(filter(
606             lambda f: f[self.POSITION][self.POSITION_INDEX] <= s[self.POSITION][self.POSITION_INDEX] <
607                       f[self.POSITION][self.POSITION_INDEX] + f[self.POSITION][self.POSITION_SPAN], frags)), sect)
608
609         def remove_duplicates(lst, keyfn, compare):
610             els = {}
611             for e in lst:
612                 eif = keyfn(e)
613                 if eif in els:
614                     if compare(els[eif], e) >= 1:
615                         continue
616                 els[eif] = e
617             return els.values()
618
619         # remove fragments with duplicated fid's and duplicated snippets
620         frags = remove_duplicates(frags, lambda f: f[self.FRAGMENT], lambda a, b: cmp(a[self.SCORE], b[self.SCORE]))
621         # frags = remove_duplicates(frags, lambda f: f[OTHER]['snippet_pos'] and f[OTHER]['snippet_pos'] or f[FRAGMENT],
622         #                           lambda a, b: cmp(a[SCORE], b[SCORE]))
623
624         # remove duplicate sections
625         sections = {}
626
627         for s in sect:
628             si = s[self.POSITION][self.POSITION_INDEX]
629             # skip existing
630             if si in sections:
631                 if sections[si]['score'] >= s[self.SCORE]:
632                     continue
633
634             m = {'score': s[self.SCORE],
635                  'section_number': s[self.POSITION][self.POSITION_INDEX] + 1,
636                  }
637             m.update(s[self.OTHER])
638             sections[si] = m
639
640         hits = sections.values()
641
642         for f in frags:
643             try:
644                 frag = catalogue.models.Fragment.objects.get(anchor=f[self.FRAGMENT], book__id=self.book_id)
645             except catalogue.models.Fragment.DoesNotExist:
646                 # stale index
647                 continue
648             # Figure out if we were searching for a token matching some word in theme name.
649             themes = frag.tags.filter(category='theme')
650             themes_hit = set()
651             if self.query_terms is not None:
652                 for i in range(0, len(f[self.OTHER]['themes'])):
653                     tms = f[self.OTHER]['themes'][i].split(r' +') + f[self.OTHER]['themes_pl'][i].split(' ')
654                     tms = map(unicode.lower, tms)
655                     for qt in self.query_terms:
656                         if qt in tms:
657                             themes_hit.add(f[self.OTHER]['themes'][i])
658                             break
659
660             def theme_by_name(n):
661                 th = filter(lambda t: t.name == n, themes)
662                 if th:
663                     return th[0]
664                 else:
665                     return None
666             themes_hit = filter(lambda a: a is not None, map(theme_by_name, themes_hit))
667
668             m = {'score': f[self.SCORE],
669                  'fragment': frag,
670                  'section_number': f[self.POSITION][self.POSITION_INDEX] + 1,
671                  'themes': themes,
672                  'themes_hit': themes_hit
673                  }
674             m.update(f[self.OTHER])
675             hits.append(m)
676
677         hits.sort(lambda a, b: cmp(a['score'], b['score']), reverse=True)
678
679         self._processed_hits = hits
680
681         return hits
682
683     @staticmethod
684     def aggregate(*result_lists):
685         books = {}
686         for rl in result_lists:
687             for r in rl:
688                 if r.book_id in books:
689                     books[r.book_id].merge(r)
690                 else:
691                     books[r.book_id] = r
692         return books.values()
693
694     def __cmp__(self, other):
695         c = cmp(self.score, other.score)
696         if c == 0:
697             # this is inverted, because earlier date is better
698             return cmp(other.published_date, self.published_date)
699         else:
700             return c
701
702     def __len__(self):
703         return len(self.hits)
704
705     def snippet_pos(self, idx=0):
706         return self.hits[idx]['snippets_pos']
707
708     def snippet_revision(self, idx=0):
709         try:
710             return self.hits[idx]['snippets_revision']
711         except (IndexError, KeyError):
712             return None
713
714
715 class Search(SolrIndex):
716     """
717     Search facilities.
718     """
719     def __init__(self, default_field="text"):
720         super(Search, self).__init__(mode='r')
721
722     def make_term_query(self, query, field='text', modal=operator.or_):
723         """
724         Returns term queries joined by boolean query.
725         modal - applies to boolean query
726         fuzzy - should the query by fuzzy.
727         """
728         if query is None:
729             query = ''
730         q = self.index.Q()
731         q = reduce(modal, map(lambda s: self.index.Q(**{field: s}), query.split(r" ")), q)
732
733         return q
734
735     def search_phrase(self, searched, field='text', book=False,
736                       filters=None,
737                       snippets=False):
738         if filters is None:
739             filters = []
740         if book:
741             filters.append(self.index.Q(is_book=True))
742
743         q = self.index.query(**{field: searched})
744         q = self.apply_filters(q, filters).field_limit(score=True, all_fields=True)
745         res = q.paginate(rows=100).execute()
746         return [SearchResult(found, how_found=u'search_phrase') for found in res]
747
748     def search_some(self, searched, fields, book=True,
749                     filters=None, snippets=True, query_terms=None):
750         assert isinstance(fields, list)
751         if filters is None:
752             filters = []
753         if book:
754             filters.append(self.index.Q(is_book=True))
755
756         query = self.index.Q()
757
758         for fld in fields:
759             query = self.index.Q(query | self.make_term_query(searched, fld))
760
761         query = self.index.query(query)
762         query = self.apply_filters(query, filters).field_limit(score=True, all_fields=True)
763         res = query.execute()
764         return [SearchResult(found, how_found='search_some', query_terms=query_terms) for found in res]
765
766     def search_everywhere(self, searched, query_terms=None):
767         """
768         Tries to use search terms to match different fields of book (or its parts).
769         E.g. one word can be an author survey, another be a part of the title, and the rest
770         are some words from third chapter.
771         """
772         books = []
773         # content only query : themes x content
774         q = self.make_term_query(searched, 'text')
775         q_themes = self.make_term_query(searched, 'themes_pl')
776
777         query = self.index.query(q).query(q_themes).field_limit(score=True, all_fields=True)
778         res = query.execute()
779
780         for found in res:
781             books.append(SearchResult(found, how_found='search_everywhere_themesXcontent', query_terms=query_terms))
782
783         # query themes/content x author/title/tags
784         in_content = self.index.Q()
785         in_meta = self.index.Q()
786
787         for fld in ['themes_pl', 'text']:
788             in_content |= self.make_term_query(searched, field=fld)
789
790         for fld in ['tags', 'authors', 'title']:
791             in_meta |= self.make_term_query(searched, field=fld)
792
793         q = in_content & in_meta
794         res = self.index.query(q).field_limit(score=True, all_fields=True).execute()
795
796         for found in res:
797             books.append(SearchResult(found, how_found='search_everywhere', query_terms=query_terms))
798
799         return books
800
801     def get_snippets(self, searchresult, query, field='text', num=1):
802         """
803         Returns a snippet for found scoreDoc.
804         """
805         maxnum = len(searchresult)
806         if num is None or num < 0 or num > maxnum:
807             num = maxnum
808         book_id = searchresult.book_id
809         revision = searchresult.snippet_revision()
810         snippets = Snippets(book_id, revision=revision)
811         snips = [None] * maxnum
812         try:
813             snippets.open()
814             idx = 0
815             while idx < maxnum and num > 0:
816                 position, length = searchresult.snippet_pos(idx)
817                 if position is None or length is None:
818                     continue
819                 text = snippets.get((int(position),
820                                      int(length)))
821                 snip = self.index.highlight(text=text, field=field, q=query)
822                 snips[idx] = snip
823                 if snip:
824                     num -= 1
825                 idx += 1
826
827         except IOError, e:
828             book = catalogue.models.Book.objects.filter(id=book_id)
829             if not book:
830                 log.error("Book does not exist for book id = %d" % book_id)
831             elif not book.get().children.exists():
832                 log.error("Cannot open snippet file for book id = %d [rev=%s], %s" % (book_id, revision, e))
833             return []
834         finally:
835             snippets.close()
836
837             # remove verse end markers..
838         snips = map(lambda s: s and s.replace("/\n", "\n"), snips)
839
840         searchresult.snippets = snips
841
842         return snips
843
844     def hint_tags(self, query, pdcounter=True, prefix=True):
845         """
846         Return auto-complete hints for tags
847         using prefix search.
848         """
849         q = self.index.Q()
850         query = query.strip()
851         for field in ['tag_name', 'tag_name_pl']:
852             if prefix:
853                 q |= self.index.Q(**{field: query + "*"})
854             else:
855                 q |= self.make_term_query(query, field=field)
856         qu = self.index.query(q)
857
858         return self.search_tags(qu, pdcounter=pdcounter)
859
860     def search_tags(self, query, filters=None, pdcounter=False):
861         """
862         Search for Tag objects using query.
863         """
864         if not filters:
865             filters = []
866         if not pdcounter:
867             filters.append(~self.index.Q(is_pdcounter=True))
868         res = self.apply_filters(query, filters).execute()
869
870         tags = []
871         pd_tags = []
872
873         for doc in res:
874             is_pdcounter = doc.get('is_pdcounter', False)
875             category = doc.get('tag_category')
876             try:
877                 if is_pdcounter:
878                     if category == 'pd_author':
879                         tag = PDCounterAuthor.objects.get(id=doc.get('tag_id'))
880                     elif category == 'pd_book':
881                         tag = PDCounterBook.objects.get(id=doc.get('tag_id'))
882                         tag.category = 'pd_book'  # make it look more lik a tag.
883                     else:
884                         # WTF
885                         print ("Warning. cannot get pdcounter tag_id=%d from db; cat=%s" % (
886                             int(doc.get('tag_id')), category)).encode('utf-8')
887                     pd_tags.append(tag)
888                 else:
889                     tag = catalogue.models.Tag.objects.get(id=doc.get("tag_id"))
890                     tags.append(tag)
891
892             except catalogue.models.Tag.DoesNotExist:
893                 pass
894             except PDCounterAuthor.DoesNotExist:
895                 pass
896             except PDCounterBook.DoesNotExist:
897                 pass
898
899         tags_slugs = set(map(lambda t: t.slug, tags))
900         tags = tags + filter(lambda t: t.slug not in tags_slugs, pd_tags)
901
902         log.debug('search_tags: %s' % tags)
903
904         return tags
905
906     def hint_books(self, query, prefix=True):
907         """
908         Returns auto-complete hints for book titles
909         Because we do not index 'pseudo' title-tags.
910         Prefix search.
911         """
912         q = self.index.Q()
913         query = query.strip()
914         if prefix:
915             q |= self.index.Q(title=query + "*")
916         else:
917             q |= self.make_term_query(query, field='title')
918         qu = self.index.query(q)
919         only_books = self.index.Q(is_book=True)
920         return self.search_books(qu, [only_books])
921
922     def search_books(self, query, filters=None, max_results=10):
923         """
924         Searches for Book objects using query
925         """
926         bks = []
927         bks_found = set()
928         query = query.query(is_book=True)
929         res = self.apply_filters(query, filters).field_limit(['book_id'])
930         for r in res:
931             try:
932                 bid = r['book_id']
933                 if bid not in bks_found:
934                     bks.append(catalogue.models.Book.objects.get(id=bid))
935                     bks_found.add(bid)
936             except catalogue.models.Book.DoesNotExist:
937                 pass
938         return bks
939
940     @staticmethod
941     def apply_filters(query, filters):
942         """
943         Apply filters to a query
944         """
945         if filters is None:
946             filters = []
947         filters = filter(lambda x: x is not None, filters)
948         for f in filters:
949             query = query.query(f)
950         return query
951
952
953 if getattr(settings, 'SEARCH_MOCK', False):
954     from .mock_search import Search