1 # -*- coding: utf-8 -*-
2 # This file is part of Wolnelektury, licensed under GNU Affero GPLv3 or later.
3 # Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
5 from sunburnt import sunburnt
9 from sunburnt import search
11 from httplib2 import socket
15 class TermVectorOptions(search.Options):
16 def __init__(self, schema, original=None):
20 self.positions = False
22 self.fields = copy.copy(original.fields)
23 self.positions = copy.copy(original.positions)
25 def update(self, positions=False, fields=None):
28 if isinstance(fields, basestring):
30 self.schema.check_fields(fields, {"stored": True})
31 self.fields.update(fields)
32 self.positions = positions
36 if self.positions or self.fields:
39 opts['tv.positions'] = 'true'
41 opts['tv.fl'] = ','.join(sorted(self.fields))
45 class CustomSolrConnection(sunburnt.SolrConnection):
46 def __init__(self, *args, **kw):
47 super(CustomSolrConnection, self).__init__(*args, **kw)
48 self.analysis_url = self.url + "analysis/field/"
50 def analyze(self, params):
51 qs = urllib.urlencode(params)
52 url = "%s?%s" % (self.analysis_url, qs)
53 if len(url) > self.max_length_get_url:
54 warnings.warn("Long query URL encountered - POSTing instead of "
55 "GETting. This query will not be cached at the HTTP layer")
56 url = self.analysis_url
60 headers={"Content-Type": "application/x-www-form-urlencoded"},
63 kwargs = dict(method="GET")
64 r, c = self.request(url, **kwargs)
66 raise sunburnt.SolrError(r, c)
70 # monkey patching sunburnt SolrSearch
71 search.SolrSearch.option_modules += ('term_vectorer',)
74 def __term_vector(self, positions=False, fields=None):
75 newself = self.clone()
76 newself.term_vectorer.update(positions, fields)
78 setattr(search.SolrSearch, 'term_vector', __term_vector)
81 def __patched__init_common_modules(self):
82 __original__init_common_modules(self)
83 self.term_vectorer = TermVectorOptions(self.schema)
84 __original__init_common_modules = search.SolrSearch._init_common_modules
85 setattr(search.SolrSearch, '_init_common_modules', __patched__init_common_modules)
88 class CustomSolrInterface(sunburnt.SolrInterface):
89 # just copied from parent and SolrConnection -> CustomSolrConnection
90 def __init__(self, url, schemadoc=None, http_connection=None, mode='', retry_timeout=-1, max_length_get_url=sunburnt.MAX_LENGTH_GET_URL):
91 self.conn = CustomSolrConnection(url, http_connection, retry_timeout, max_length_get_url)
92 self.schemadoc = schemadoc
94 self.writeable = False
99 except socket.error, e:
100 raise socket.error, "Cannot connect to Solr server, and search indexing is enabled (%s)" % str(e)
102 def _analyze(self, **kwargs):
103 if not self.readable:
104 raise TypeError("This Solr instance is only for writing")
106 'analysis_showmatch': True
108 if 'field' in kwargs: args['analysis_fieldname'] = kwargs['field']
109 if 'text' in kwargs: args['analysis_fieldvalue'] = kwargs['text']
110 if 'q' in kwargs: args['q'] = kwargs['q']
111 if 'query' in kwargs: args['q'] = kwargs['q']
113 params = map(lambda (k, v): (k.replace('_', '.'), v), sunburnt.params_from_dict(**args))
115 content = self.conn.analyze(params)
116 doc = etree.fromstring(content)
119 def highlight(self, **kwargs):
120 doc = self._analyze(**kwargs)
121 analyzed = doc.xpath("//lst[@name='index']/arr[last()]/lst[bool/@name='match']")
124 start = int(wrd.xpath("int[@name='start']")[0].text)
125 end = int(wrd.xpath("int[@name='end']")[0].text)
126 matches.add((start, end))
129 return self.substring(kwargs['text'], matches,
130 margins=kwargs.get('margins', 30),
131 mark=kwargs.get('mark', ("<b>", "</b>")))
135 def analyze(self, **kwargs):
136 doc = self._analyze(**kwargs)
137 terms = doc.xpath("//lst[@name='index']/arr[last()]/lst/str[1]")
138 terms = map(lambda n: unicode(n.text), terms)
141 def expand_margins(self, text, start, end):
145 ws = re.compile(r"\W", re.UNICODE)
146 return bool(ws.match(x))
149 if is_boundary(text[start - 1]):
153 while end < totlen - 1:
154 if is_boundary(text[end + 1]):
160 def substring(self, text, matches, margins=30, mark=("<b>", "</b>")):
164 matches_margins = map(lambda (s, e):
166 (max(0, s - margins), min(totlen, e + margins))),
168 matches_margins = map(lambda (m, (s, e)):
169 (m, self.expand_margins(text, s, e)),
172 # lets start with first match
173 (start, end) = matches_margins[0][1]
174 matches = [matches_margins[0][0]]
176 for (m, (s, e)) in matches_margins[1:]:
177 if end < s or start > e:
179 start = min(start, s)
183 snip = text[start:end]
184 matches.sort(lambda a, b: cmp(b[0], a[0]))
186 for (s, e) in matches:
188 snip = snip[:e + off] + mark[1] + snip[e + off:]
189 snip = snip[:s + off] + mark[0] + snip[s + off:]