7f8bf9f6a29adf789e3bda480d2814cc7fe311d1
[wolnelektury.git] / apps / search / index.py
1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
4 #
5 from django.conf import settings
6
7 import os
8 import re
9 import errno
10 from librarian import dcparser
11 from librarian.parser import WLDocument
12 from lxml import etree
13 import catalogue.models
14 from pdcounter.models import Author as PDCounterAuthor, BookStub as PDCounterBook
15 from itertools import chain
16 import traceback
17 import logging
18 log = logging.getLogger('search')
19 import sunburnt
20 import custom
21 import operator
22
23 log = logging.getLogger('search')
24
25 class SolrIndex(object):
26     def __init__(self, mode=None):
27         self.index = custom.CustomSolrInterface(settings.SOLR, mode=mode)
28
29
30 class Snippets(object):
31     """
32     This class manages snippet files for indexed object (book)
33     the snippets are concatenated together, and their positions and
34     lengths are kept in lucene index fields.
35     """
36     SNIPPET_DIR = "snippets"
37
38     def __init__(self, book_id, revision=None):
39         try:
40             os.makedirs(os.path.join(settings.SEARCH_INDEX, self.SNIPPET_DIR))
41         except OSError as exc:
42             if exc.errno == errno.EEXIST:
43                 pass
44             else: raise
45         self.book_id = book_id
46         self.revision = revision
47         self.file = None
48
49     @property
50     def path(self):
51         if self.revision: fn = "%d.%d" % (self.book_id, self.revision)
52         else: fn = "%d" % self.book_id
53
54         return os.path.join(settings.SEARCH_INDEX, self.SNIPPET_DIR, fn)
55
56     def open(self, mode='r'):
57         """
58         Open the snippet file. Call .close() afterwards.
59         """
60         if not 'b' in mode:
61             mode += 'b'
62
63         if 'w' in mode:
64             if os.path.exists(self.path):
65                 self.revision = 1
66                 while True:
67                     if not os.path.exists(self.path):
68                         break
69                     self.revision += 1
70
71         self.file = open(self.path, mode)
72         self.position = 0
73         return self
74
75     def add(self, snippet):
76         """
77         Append a snippet (unicode) to the snippet file.
78         Return a (position, length) tuple
79         """
80         txt = snippet.encode('utf-8')
81         l = len(txt)
82         self.file.write(txt)
83         pos = (self.position, l)
84         self.position += l
85         return pos
86
87     def get(self, pos):
88         """
89         Given a tuple of (position, length) return an unicode
90         of the snippet stored there.
91         """
92         self.file.seek(pos[0], 0)
93         txt = self.file.read(pos[1]).decode('utf-8')
94         return txt
95
96     def close(self):
97         """Close snippet file"""
98         if self.file:
99             self.file.close()
100
101     def remove(self):
102         self.revision = None
103         try:
104             os.unlink(self.path)
105             self.revision = 0
106             while True:
107                 self.revision += 1
108                 os.unlink(self.path)
109         except OSError:
110             pass
111
112
113 class Index(SolrIndex):
114     """
115     Class indexing books.
116     """
117     def __init__(self):
118         super(Index, self).__init__(mode='rw')
119
120     def delete_query(self, *queries):
121         """
122         index.delete(queries=...) doesn't work, so let's reimplement it
123         using deletion of list of uids.
124         """
125         uids = set()
126         for q in queries:
127             if isinstance(q, sunburnt.search.LuceneQuery):
128                 q = self.index.query(q)
129             q.field_limiter.update(['uid'])
130             st = 0
131             rows = 100
132             while True:
133                 ids = q.paginate(start=st, rows=rows).execute()
134                 if not len(ids):
135                     break
136                 for res in ids:
137                     uids.add(res['uid'])
138                 st += rows
139         if uids:
140             self.index.delete(uids)
141             return True
142         else:
143             return False
144
145     def index_tags(self, *tags, **kw):
146         """
147         Re-index global tag list.
148         Removes all tags from index, then index them again.
149         Indexed fields include: id, name (with and without polish stems), category
150         """
151         log.debug("Indexing tags")
152         remove_only = kw.get('remove_only', False)
153         # first, remove tags from index.
154         if tags:
155             tag_qs = []
156             for tag in tags:
157                 q_id = self.index.Q(tag_id=tag.id)
158
159                 if isinstance(tag, PDCounterAuthor):
160                     q_cat = self.index.Q(tag_category='pd_author')
161                 elif isinstance(tag, PDCounterBook):
162                     q_cat = self.index.Q(tag_category='pd_book')
163                 else:
164                     q_cat = self.index.Q(tag_category=tag.category)
165
166                 q_id_cat = self.index.Q(q_id & q_cat)
167                 tag_qs.append(q_id_cat)
168             self.delete_query(*tag_qs)
169         else:  # all
170             q = self.index.Q(tag_id__any=True)
171             self.delete_query(q)
172
173         if not remove_only:
174             # then add them [all or just one passed]
175             if not tags:
176                 tags = chain(catalogue.models.Tag.objects.exclude(category='set'), \
177                     PDCounterAuthor.objects.all(), \
178                     PDCounterBook.objects.all())
179
180             for tag in tags:
181                 if isinstance(tag, PDCounterAuthor):
182                     doc = {
183                         "tag_id": int(tag.id),
184                         "tag_name": tag.name,
185                         "tag_name_pl": tag.name,
186                         "tag_category": 'pd_author',
187                         "is_pdcounter": True,
188                         "uid": "tag%d_pd_a" % tag.id
189                         }
190                 elif isinstance(tag, PDCounterBook):
191                     doc = {
192                         "tag_id": int(tag.id),
193                         "tag_name": tag.title,
194                         "tag_name_pl": tag.title,
195                         "tag_category": 'pd_book',
196                         "is_pdcounter": True,
197                         "uid": "tag%d_pd_b" % tag.id
198                         }
199                 else:
200                     doc = {
201                         "tag_id": int(tag.id),
202                         "tag_name": tag.name,
203                         "tag_name_pl": tag.name,
204                         "tag_category": tag.category,
205                         "is_pdcounter": False,
206                         "uid": "tag%d" % tag.id
207                         }
208                 self.index.add(doc)
209
210     def create_book_doc(self, book):
211         """
212         Create a lucene document referring book id.
213         """
214         doc = {
215             'book_id': int(book.id),
216             }
217         if book.parent is not None:
218             doc["parent_id"] = int(book.parent.id)
219         return doc
220
221     def remove_book(self, book_or_id, remove_snippets=True):
222         """Removes a book from search index.
223         book - Book instance."""
224         if isinstance(book_or_id, catalogue.models.Book):
225             book_id = book_or_id.id
226         else:
227             book_id = book_or_id
228
229         self.delete_query(self.index.Q(book_id=book_id))
230
231         if remove_snippets:
232             snippets = Snippets(book_id)
233             snippets.remove()
234
235     def index_book(self, book, book_info=None, overwrite=True):
236         """
237         Indexes the book.
238         Creates a lucene document for extracted metadata
239         and calls self.index_content() to index the contents of the book.
240         """
241         if overwrite:
242             # we don't remove snippets, since they might be still needed by
243             # threads using not reopened index
244             self.remove_book(book, remove_snippets=False)
245
246         book_doc = self.create_book_doc(book)
247         meta_fields = self.extract_metadata(book, book_info, dc_only=['source_name', 'authors', 'translators', 'title'])
248         # let's not index it - it's only used for extracting publish date
249         if 'source_name' in meta_fields:
250             del meta_fields['source_name']
251
252         for n, f in meta_fields.items():
253             book_doc[n] = f
254
255         book_doc['uid'] = "book%s" % book_doc['book_id']
256         self.index.add(book_doc)
257         del book_doc
258         book_fields = {
259             'title': meta_fields['title'],
260             'authors': meta_fields['authors'],
261             'published_date': meta_fields['published_date']
262             }
263
264         if 'translators' in meta_fields:
265             book_fields['translators'] = meta_fields['translators']
266
267         self.index_content(book, book_fields=book_fields)
268
269     master_tags = [
270         'opowiadanie',
271         'powiesc',
272         'dramat_wierszowany_l',
273         'dramat_wierszowany_lp',
274         'dramat_wspolczesny', 'liryka_l', 'liryka_lp',
275         'wywiad',
276         ]
277
278     ignore_content_tags = [
279         'uwaga', 'extra',
280         'zastepnik_tekstu', 'sekcja_asterysk', 'separator_linia', 'zastepnik_wersu',
281         'didaskalia',
282         'naglowek_aktu', 'naglowek_sceny', 'naglowek_czesc',
283         ]
284
285     footnote_tags = ['pa', 'pt', 'pr', 'pe']
286
287     skip_header_tags = ['autor_utworu', 'nazwa_utworu', 'dzielo_nadrzedne', '{http://www.w3.org/1999/02/22-rdf-syntax-ns#}RDF']
288
289     published_date_re = re.compile("([0-9]+)[\]. ]*$")
290
291     def extract_metadata(self, book, book_info=None, dc_only=None):
292         """
293         Extract metadata from book and returns a map of fields keyed by fieldname
294         """
295         fields = {}
296
297         if book_info is None:
298             book_info = dcparser.parse(open(book.xml_file.path))
299
300         fields['slug'] = book.slug
301         fields['tags'] = [t.name  for t in book.tags]
302         fields['is_book'] = True
303
304         # validator, name
305         for field in dcparser.BookInfo.FIELDS:
306             if dc_only and field.name not in dc_only:
307                 continue
308             if hasattr(book_info, field.name):
309                 if not getattr(book_info, field.name):
310                     continue
311                 # since no type information is available, we use validator
312                 type_indicator = field.validator
313                 if type_indicator == dcparser.as_unicode:
314                     s = getattr(book_info, field.name)
315                     if field.multiple:
316                         s = ', '.join(s)
317                     fields[field.name] = s
318                 elif type_indicator == dcparser.as_person:
319                     p = getattr(book_info, field.name)
320                     if isinstance(p, dcparser.Person):
321                         persons = unicode(p)
322                     else:
323                         persons = ', '.join(map(unicode, p))
324                     fields[field.name] = persons
325                 elif type_indicator == dcparser.as_date:
326                     dt = getattr(book_info, field.name)
327                     fields[field.name] = dt
328
329         # get published date
330         pd = None
331         if hasattr(book_info, 'source_name') and book_info.source_name:
332             match = self.published_date_re.search(book_info.source_name)
333             if match is not None:
334                 pd = str(match.groups()[0])
335         if not pd: pd = ""
336         fields["published_date"] = pd
337
338         return fields
339
340     # def add_gaps(self, fields, fieldname):
341     #     """
342     #     Interposes a list of fields with gap-fields, which are indexed spaces and returns it.
343     #     This allows for doing phrase queries which do not overlap the gaps (when slop is 0).
344     #     """
345     #     def gap():
346     #         while True:
347     #             yield Field(fieldname, ' ', Field.Store.NO, Field.Index.NOT_ANALYZED)
348     #     return reduce(lambda a, b: a + b, zip(fields, gap()))[0:-1]
349
350     def get_master(self, root):
351         """
352         Returns the first master tag from an etree.
353         """
354         for master in root.iter():
355             if master.tag in self.master_tags:
356                 return master
357
358     def index_content(self, book, book_fields={}):
359         """
360         Walks the book XML and extract content from it.
361         Adds parts for each header tag and for each fragment.
362         """
363         wld = WLDocument.from_file(book.xml_file.path, parse_dublincore=False)
364         root = wld.edoc.getroot()
365
366         master = self.get_master(root)
367         if master is None:
368             return []
369
370         def walker(node, ignore_tags=[]):
371
372             if node.tag not in ignore_tags:
373                 yield node, None, None
374                 if node.text is not None:
375                     yield None, node.text, None
376                 for child in list(node):
377                     for b, t, e in walker(child):
378                         yield b, t, e
379                 yield None, None, node
380
381             if node.tail is not None:
382                 yield None, node.tail, None
383             return
384
385         def fix_format(text):
386             #            separator = [u" ", u"\t", u".", u";", u","]
387             if isinstance(text, list):
388                 # need to join it first
389                 text = filter(lambda s: s is not None, content)
390                 text = u' '.join(text)
391                 # for i in range(len(text)):
392                 #     if i > 0:
393                 #         if text[i][0] not in separator\
394                 #             and text[i - 1][-1] not in separator:
395                 #          text.insert(i, u" ")
396
397             return re.sub("(?m)/$", "", text)
398
399         def add_part(snippets, **fields):
400             doc = self.create_book_doc(book)
401             for n, v in book_fields.items():
402                 doc[n] = v
403
404             doc['header_index'] = fields["header_index"]
405             doc['header_span'] = 'header_span' in fields and fields['header_span'] or 1
406             doc['header_type'] = fields['header_type']
407
408             doc['text'] = fields['text']
409
410             # snippets
411             snip_pos = snippets.add(fields["text"])
412
413             doc['snippets_position'] = snip_pos[0]
414             doc['snippets_length'] = snip_pos[1]
415             if snippets.revision:
416                 doc["snippets_revision"] = snippets.revision
417
418             if 'fragment_anchor' in fields:
419                 doc["fragment_anchor"] = fields['fragment_anchor']
420
421             if 'themes' in fields:
422                 doc['themes'] = fields['themes']
423             doc['uid'] = "part%s%s%s" % (doc['header_index'],
424                                          doc['header_span'],
425                                          doc.get('fragment_anchor', ''))
426             return doc
427
428         def give_me_utf8(s):
429             if isinstance(s, unicode):
430                 return s.encode('utf-8')
431             else:
432                 return s
433
434         fragments = {}
435         snippets = Snippets(book.id).open('w')
436         try:
437             for header, position in zip(list(master), range(len(master))):
438
439                 if header.tag in self.skip_header_tags:
440                     continue
441                 if header.tag is etree.Comment:
442                     continue
443
444                 # section content
445                 content = []
446                 footnote = []
447
448                 def all_content(text):
449                     for frag in fragments.values():
450                         frag['text'].append(text)
451                     content.append(text)
452                 handle_text = [all_content]
453
454                 for start, text, end in walker(header, ignore_tags=self.ignore_content_tags):
455                     # handle footnotes
456                     if start is not None and start.tag in self.footnote_tags:
457                         footnote = []
458
459                         def collect_footnote(t):
460                             footnote.append(t)
461
462                         handle_text.append(collect_footnote)
463                     elif end is not None and footnote is not [] and end.tag in self.footnote_tags:
464                         handle_text.pop()
465                         doc = add_part(snippets, header_index=position, header_type=header.tag,
466                                        text=u''.join(footnote),
467                                        is_footnote=True)
468                         self.index.add(doc)
469                         footnote = []
470
471                     # handle fragments and themes.
472                     if start is not None and start.tag == 'begin':
473                         fid = start.attrib['id'][1:]
474                         fragments[fid] = {'text': [], 'themes': [], 'start_section': position, 'start_header': header.tag}
475
476                     # themes for this fragment
477                     elif start is not None and start.tag == 'motyw':
478                         fid = start.attrib['id'][1:]
479                         handle_text.append(None)
480                         if start.text is not None:
481                             fragments[fid]['themes'] += map(unicode.strip, map(unicode, (start.text.split(','))))
482                     elif end is not None and end.tag == 'motyw':
483                         handle_text.pop()
484
485                     elif start is not None and start.tag == 'end':
486                         fid = start.attrib['id'][1:]
487                         if fid not in fragments:
488                             continue  # a broken <end> node, skip it
489                         frag = fragments[fid]
490                         if frag['themes'] == []:
491                             continue  # empty themes list.
492                         del fragments[fid]
493
494                         doc = add_part(snippets,
495                                        header_type=frag['start_header'],
496                                        header_index=frag['start_section'],
497                                        header_span=position - frag['start_section'] + 1,
498                                        fragment_anchor=fid,
499                                        text=fix_format(frag['text']),
500                                        themes=frag['themes'])
501                         self.index.add(doc)
502
503                         # Collect content.
504
505                     if text is not None and handle_text is not []:
506                         hdl = handle_text[-1]
507                         if hdl is not None:
508                             hdl(text)
509
510                         # in the end, add a section text.
511                 doc = add_part(snippets, header_index=position,
512                                header_type=header.tag, text=fix_format(content))
513
514                 self.index.add(doc)
515
516         finally:
517             snippets.close()
518
519
520 class SearchResult(object):
521     def __init__(self, doc, how_found=None, query=None, query_terms=None):
522         #        self.search = search
523         self.boost = 1.0
524         self._hits = []
525         self._processed_hits = None  # processed hits
526         self.snippets = []
527         self.query_terms = query_terms
528
529         if 'score' in doc:
530             self._score = doc['score']
531         else:
532             self._score = 0
533
534         self.book_id = int(doc["book_id"])
535
536         try:
537             self.published_date = int(doc.get("published_date"))
538         except ValueError:
539             self.published_date = 0
540
541         # content hits
542         header_type = doc.get("header_type", None)
543         # we have a content hit in some header of fragment
544         if header_type is not None:
545             sec = (header_type, int(doc["header_index"]))
546             header_span = doc['header_span']
547             header_span = header_span is not None and int(header_span) or 1
548             fragment = doc.get("fragment_anchor", None)
549             snippets_pos = (doc['snippets_position'], doc['snippets_length'])
550             snippets_rev = doc.get('snippets_revision', None)
551
552             hit = (sec + (header_span,), fragment, self._score, {
553                 'how_found': how_found,
554                 'snippets_pos': snippets_pos,
555                 'snippets_revision': snippets_rev,
556                 'themes': doc.get('themes', []),
557                 'themes_pl': doc.get('themes_pl', [])
558                 })
559
560             self._hits.append(hit)
561
562     def __unicode__(self):
563         return u"<SR id=%d %d(%d) hits score=%f %d snippets>" % \
564             (self.book_id, len(self._hits), self._processed_hits and len(self._processed_hits) or -1, self._score, len(self.snippets))
565
566     def __str__(self):
567         return unicode(self).encode('utf-8')
568
569     @property
570     def score(self):
571         return self._score * self.boost
572
573     def merge(self, other):
574         if self.book_id != other.book_id:
575             raise ValueError("this search result is or book %d; tried to merge with %d" % (self.book_id, other.book_id))
576         self._hits += other._hits
577         if other.score > self.score:
578             self._score = other._score
579         return self
580
581     def get_book(self):
582         if hasattr(self, '_book'):
583             return self._book
584         self._book = catalogue.models.Book.objects.get(id=self.book_id)
585         return self._book
586
587     book = property(get_book)
588
589     POSITION = 0
590     FRAGMENT = 1
591     POSITION_INDEX = 1
592     POSITION_SPAN = 2
593     SCORE = 2
594     OTHER = 3
595
596     @property
597     def hits(self):
598         if self._processed_hits is not None:
599             return self._processed_hits
600
601         # to sections and fragments
602         frags = filter(lambda r: r[self.FRAGMENT] is not None, self._hits)
603
604         sect = filter(lambda r: r[self.FRAGMENT] is None, self._hits)
605
606         # sections not covered by fragments
607         sect = filter(lambda s: 0 == len(filter(
608             lambda f: s[self.POSITION][self.POSITION_INDEX] >= f[self.POSITION][self.POSITION_INDEX]
609             and s[self.POSITION][self.POSITION_INDEX] < f[self.POSITION][self.POSITION_INDEX] + f[self.POSITION][self.POSITION_SPAN],
610             frags)), sect)
611
612         hits = []
613
614         def remove_duplicates(lst, keyfn, compare):
615             els = {}
616             for e in lst:
617                 eif = keyfn(e)
618                 if eif in els:
619                     if compare(els[eif], e) >= 1:
620                         continue
621                 els[eif] = e
622             return els.values()
623
624         # remove fragments with duplicated fid's and duplicated snippets
625         frags = remove_duplicates(frags, lambda f: f[self.FRAGMENT], lambda a, b: cmp(a[self.SCORE], b[self.SCORE]))
626         # frags = remove_duplicates(frags, lambda f: f[OTHER]['snippet_pos'] and f[OTHER]['snippet_pos'] or f[FRAGMENT],
627         #                           lambda a, b: cmp(a[SCORE], b[SCORE]))
628
629         # remove duplicate sections
630         sections = {}
631
632         for s in sect:
633             si = s[self.POSITION][self.POSITION_INDEX]
634             # skip existing
635             if si in sections:
636                 if sections[si]['score'] >= s[self.SCORE]:
637                     continue
638
639             m = {'score': s[self.SCORE],
640                  'section_number': s[self.POSITION][self.POSITION_INDEX] + 1,
641                  }
642             m.update(s[self.OTHER])
643             sections[si] = m
644
645         hits = sections.values()
646
647         for f in frags:
648             try:
649                 frag = catalogue.models.Fragment.objects.get(anchor=f[self.FRAGMENT], book__id=self.book_id)
650             except catalogue.models.Fragment.DoesNotExist:
651                 # stale index
652                 continue
653             # Figure out if we were searching for a token matching some word in theme name.
654             themes = frag.tags.filter(category='theme')
655             themes_hit = set()
656             if self.query_terms is not None:
657                 for i in range(0, len(f[self.OTHER]['themes'])):
658                     tms = f[self.OTHER]['themes'][i].split(r' +') + f[self.OTHER]['themes_pl'][i].split(' ')
659                     tms = map(unicode.lower, tms)
660                     for qt in self.query_terms:
661                         if qt in tms:
662                             themes_hit.add(f[self.OTHER]['themes'][i])
663                             break
664
665             def theme_by_name(n):
666                 th = filter(lambda t: t.name == n, themes)
667                 if th:
668                     return th[0]
669                 else:
670                     return None
671             themes_hit = filter(lambda a: a is not None, map(theme_by_name, themes_hit))
672
673             m = {'score': f[self.SCORE],
674                  'fragment': frag,
675                  'section_number': f[self.POSITION][self.POSITION_INDEX] + 1,
676                  'themes': themes,
677                  'themes_hit': themes_hit
678                  }
679             m.update(f[self.OTHER])
680             hits.append(m)
681
682         hits.sort(lambda a, b: cmp(a['score'], b['score']), reverse=True)
683
684         self._processed_hits = hits
685
686         return hits
687
688     @staticmethod
689     def aggregate(*result_lists):
690         books = {}
691         for rl in result_lists:
692             for r in rl:
693                 if r.book_id in books:
694                     books[r.book_id].merge(r)
695                 else:
696                     books[r.book_id] = r
697         return books.values()
698
699     def __cmp__(self, other):
700         c = cmp(self.score, other.score)
701         if c == 0:
702             # this is inverted, because earlier date is better
703             return cmp(other.published_date, self.published_date)
704         else:
705             return c
706
707     def __len__(self):
708         return len(self.hits)
709
710     def snippet_pos(self, idx=0):
711         return self.hits[idx]['snippets_pos']
712
713     def snippet_revision(self, idx=0):
714         try:
715             return self.hits[idx]['snippets_revision']
716         except:
717             return None
718
719
720 class Search(SolrIndex):
721     """
722     Search facilities.
723     """
724     def __init__(self, default_field="text"):
725         super(Search, self).__init__(mode='r')
726
727
728     def make_term_query(self, query, field='text', modal=operator.or_):
729         """
730         Returns term queries joined by boolean query.
731         modal - applies to boolean query
732         fuzzy - should the query by fuzzy.
733         """
734         if query is None: query = ''
735         q = self.index.Q()
736         q = reduce(modal, map(lambda s: self.index.Q(**{field: s}),
737                         query.split(r" ")), q)
738
739         return q
740
741     def search_phrase(self, searched, field='text', book=False,
742                       filters=None,
743                       snippets=False):
744         if filters is None: filters = []
745         if book: filters.append(self.index.Q(is_book=True))
746
747         q = self.index.query(**{field: searched})
748         q = self.apply_filters(q, filters).field_limit(score=True, all_fields=True)
749         res = q.execute()
750         return [SearchResult(found, how_found=u'search_phrase') for found in res]
751
752     def search_some(self, searched, fields, book=True,
753                     filters=None, snippets=True, query_terms=None):
754         assert isinstance(fields, list)
755         if filters is None: filters = []
756         if book: filters.append(self.index.Q(is_book=True))
757
758         query = self.index.Q()
759
760         for fld in fields:
761             query = self.index.Q(query | self.make_term_query(searched, fld))
762
763         query = self.index.query(query)
764         query = self.apply_filters(query, filters).field_limit(score=True, all_fields=True)
765         res = query.execute()
766         return [SearchResult(found, how_found='search_some', query_terms=query_terms) for found in res]
767
768
769     def search_everywhere(self, searched, query_terms=None):
770         """
771         Tries to use search terms to match different fields of book (or its parts).
772         E.g. one word can be an author survey, another be a part of the title, and the rest
773         are some words from third chapter.
774         """
775         books = []
776         # content only query : themes x content
777         q = self.make_term_query(searched, 'text')
778         q_themes = self.make_term_query(searched, 'themes_pl')
779
780         query = self.index.query(q).query(q_themes).field_limit(score=True, all_fields=True)
781         res = query.execute()
782
783         for found in res:
784             books.append(SearchResult(found, how_found='search_everywhere_themesXcontent', query_terms=query_terms))
785
786         # query themes/content x author/title/tags
787         in_content = self.index.Q()
788         in_meta = self.index.Q()
789
790         for fld in ['themes_pl', 'text']:
791             in_content |= self.make_term_query(searched, field=fld)
792
793         for fld in ['tags', 'authors', 'title']:
794             in_meta |= self.make_term_query(searched, field=fld)
795
796         q = in_content & in_meta
797         res = self.index.query(q).field_limit(score=True, all_fields=True).execute()
798
799         for found in res:
800             books.append(SearchResult(found, how_found='search_everywhere', query_terms=query_terms))
801
802         return books
803
804     def get_snippets(self, searchresult, query, field='text', num=1):
805         """
806         Returns a snippet for found scoreDoc.
807         """
808         maxnum = len(searchresult)
809         if num is None or num < 0 or num > maxnum:
810             num = maxnum
811         book_id = searchresult.book_id
812         revision = searchresult.snippet_revision()
813         snippets = Snippets(book_id, revision=revision)
814         snips = [None] * maxnum
815         try:
816             snippets.open()
817             idx = 0
818             while idx < maxnum and num > 0:
819                 position, length = searchresult.snippet_pos(idx)
820                 if position is None or length is None:
821                     continue
822                 text = snippets.get((int(position),
823                                      int(length)))
824                 snip = self.index.highlight(text=text, field=field, q=query)
825                 snips[idx] = snip
826                 if snip:
827                     num -= 1
828                 idx += 1
829
830         except IOError, e:
831             log.error("Cannot open snippet file for book id = %d [rev=%s], %s" % (book_id, revision, e))
832             return []
833         finally:
834             snippets.close()
835
836             # remove verse end markers..
837         snips = map(lambda s: s and s.replace("/\n", "\n"), snips)
838
839         searchresult.snippets = snips
840
841         return snips
842
843     def hint_tags(self, query, pdcounter=True, prefix=True):
844         """
845         Return auto-complete hints for tags
846         using prefix search.
847         """
848         q = self.index.Q()
849         query = query.strip()
850         for field in ['tag_name', 'tag_name_pl']:
851             if prefix:
852                 q |= self.index.Q(**{field: query + "*"})
853             else:
854                 q |= self.make_term_query(query, field=field)
855         qu = self.index.query(q)
856
857         return self.search_tags(qu, pdcounter=pdcounter)
858
859     def search_tags(self, query, filters=None, pdcounter=False):
860         """
861         Search for Tag objects using query.
862         """
863         if not filters: filters = []
864         if not pdcounter:
865             filters.append(~self.index.Q(is_pdcounter=True))
866         res = self.apply_filters(query, filters).execute()
867
868         tags = []
869         pd_tags = []
870
871         for doc in res:
872             is_pdcounter = doc.get('is_pdcounter', False)
873             category = doc.get('tag_category')
874             try:
875                 if is_pdcounter == True:
876                     if category == 'pd_author':
877                         tag = PDCounterAuthor.objects.get(id=doc.get('tag_id'))
878                     elif category == 'pd_book':
879                         tag = PDCounterBook.objects.get(id=doc.get('tag_id'))
880                         tag.category = 'pd_book'  # make it look more lik a tag.
881                     else:
882                         print ("Warning. cannot get pdcounter tag_id=%d from db; cat=%s" % (int(doc.get('tag_id')), category)).encode('utf-8')
883                     pd_tags.append(tag)
884                 else:
885                     tag = catalogue.models.Tag.objects.get(id=doc.get("tag_id"))
886                     tags.append(tag)
887
888             except catalogue.models.Tag.DoesNotExist: pass
889             except PDCounterAuthor.DoesNotExist: pass
890             except PDCounterBook.DoesNotExist: pass
891
892         tags_slugs = set(map(lambda t: t.slug, tags))
893         tags = tags + filter(lambda t: not t.slug in tags_slugs, pd_tags)
894
895         log.debug('search_tags: %s' % tags)
896
897         return tags
898
899     def hint_books(self, query, prefix=True):
900         """
901         Returns auto-complete hints for book titles
902         Because we do not index 'pseudo' title-tags.
903         Prefix search.
904         """
905         q = self.index.Q()
906         query = query.strip()
907         if prefix:
908             q |= self.index.Q(title=query + "*")
909         else:
910             q |= self.make_term_query(query, field='title')
911         qu = self.index.query(q)
912         only_books = self.index.Q(is_book=True)
913         return self.search_books(qu, [only_books])
914
915     def search_books(self, query, filters=None, max_results=10):
916         """
917         Searches for Book objects using query
918         """
919         bks = []
920         bks_found = set()
921         query = query.query(is_book=True)
922         res = self.apply_filters(query, filters).field_limit(['book_id'])
923         for r in res:
924             try:
925                 bid = r['book_id']
926                 if not bid in bks_found:
927                     bks.append(catalogue.models.Book.objects.get(id=bid))
928                     bks_found.add(bid)
929             except catalogue.models.Book.DoesNotExist: pass
930         return bks
931  
932
933     @staticmethod
934     def apply_filters(query, filters):
935         """
936         Apply filters to a query
937         """
938         if filters is None: filters = []
939         filters = filter(lambda x: x is not None, filters)
940         for f in filters:
941             query = query.query(f)
942         return query