+++ /dev/null
-import select
-import socket
-import time
-import struct
-import warnings
-import operator
-import apis.current as sphinxapi
-
-try:
- import decimal
-except ImportError:
- from django.utils import _decimal as decimal # for Python 2.3
-
-from django.db.models.query import QuerySet, Q
-from django.conf import settings
-
-__all__ = ('SearchError', 'ConnectionError', 'SphinxSearch', 'SphinxRelation')
-
-from django.contrib.contenttypes.models import ContentType
-from datetime import datetime, date
-
-# server settings
-SPHINX_SERVER = getattr(settings, 'SPHINX_SERVER', 'localhost')
-SPHINX_PORT = int(getattr(settings, 'SPHINX_PORT', 3312))
-
-# These require search API 275 (Sphinx 0.9.8)
-SPHINX_RETRIES = int(getattr(settings, 'SPHINX_RETRIES', 0))
-SPHINX_RETRIES_DELAY = int(getattr(settings, 'SPHINX_RETRIES_DELAY', 5))
-
-MAX_INT = int(2**31-1)
-
-class SearchError(Exception): pass
-class ConnectionError(Exception): pass
-
-class SphinxProxy(object):
- """
- Acts exactly like a normal instance of an object except that
- it will handle any special sphinx attributes in a _sphinx class.
- """
- __slots__ = ('__dict__', '__instance__', '_sphinx')
-
- def __init__(self, instance, attributes):
- object.__setattr__(self, '__instance__', instance)
- object.__setattr__(self, '_sphinx', attributes)
-
- def _get_current_object(self):
- """
- Return the current object. This is useful if you want the real object
- behind the proxy at a time for performance reasons or because you want
- to pass the object into a different context.
- """
- return self.__instance__
- __current_object = property(_get_current_object)
-
- def __dict__(self):
- try:
- return self.__current_object.__dict__
- except RuntimeError:
- return AttributeError('__dict__')
- __dict__ = property(__dict__)
-
- def __repr__(self):
- try:
- obj = self.__current_object
- except RuntimeError:
- return '<%s unbound>' % self.__class__.__name__
- return repr(obj)
-
- def __nonzero__(self):
- try:
- return bool(self.__current_object)
- except RuntimeError:
- return False
-
- def __unicode__(self):
- try:
- return unicode(self.__current_oject)
- except RuntimeError:
- return repr(self)
-
- def __dir__(self):
- try:
- return dir(self.__current_object)
- except RuntimeError:
- return []
-
- def __getattr__(self, name, value=None):
- if name == '__members__':
- return dir(self.__current_object)
- elif name == '_sphinx':
- return object.__getattr__(self, '_sphinx', value)
- return getattr(self.__current_object, name)
-
- def __setattr__(self, name, value):
- if name == '_sphinx':
- return object.__setattr__(self, '_sphinx', value)
- return setattr(self.__current_object, name, value)
-
- def __setitem__(self, key, value):
- self.__current_object[key] = value
-
- def __delitem__(self, key):
- del self.__current_object[key]
-
- def __setslice__(self, i, j, seq):
- self.__current_object[i:j] = seq
-
- def __delslice__(self, i, j):
- del self.__current_object[i:j]
-
- __delattr__ = lambda x, n: delattr(x.__current_object, n)
- __str__ = lambda x: str(x.__current_object)
- __unicode__ = lambda x: unicode(x.__current_object)
- __lt__ = lambda x, o: x.__current_object < o
- __le__ = lambda x, o: x.__current_object <= o
- __eq__ = lambda x, o: x.__current_object == o
- __ne__ = lambda x, o: x.__current_object != o
- __gt__ = lambda x, o: x.__current_object > o
- __ge__ = lambda x, o: x.__current_object >= o
- __cmp__ = lambda x, o: cmp(x.__current_object, o)
- __hash__ = lambda x: hash(x.__current_object)
- # attributes are currently not callable
- # __call__ = lambda x, *a, **kw: x.__current_object(*a, **kw)
- __len__ = lambda x: len(x.__current_object)
- __getitem__ = lambda x, i: x.__current_object[i]
- __iter__ = lambda x: iter(x.__current_object)
- __contains__ = lambda x, i: i in x.__current_object
- __getslice__ = lambda x, i, j: x.__current_object[i:j]
- __add__ = lambda x, o: x.__current_object + o
- __sub__ = lambda x, o: x.__current_object - o
- __mul__ = lambda x, o: x.__current_object * o
- __floordiv__ = lambda x, o: x.__current_object // o
- __mod__ = lambda x, o: x.__current_object % o
- __divmod__ = lambda x, o: x.__current_object.__divmod__(o)
- __pow__ = lambda x, o: x.__current_object ** o
- __lshift__ = lambda x, o: x.__current_object << o
- __rshift__ = lambda x, o: x.__current_object >> o
- __and__ = lambda x, o: x.__current_object & o
- __xor__ = lambda x, o: x.__current_object ^ o
- __or__ = lambda x, o: x.__current_object | o
- __div__ = lambda x, o: x.__current_object.__div__(o)
- __truediv__ = lambda x, o: x.__current_object.__truediv__(o)
- __neg__ = lambda x: -(x.__current_object)
- __pos__ = lambda x: +(x.__current_object)
- __abs__ = lambda x: abs(x.__current_object)
- __invert__ = lambda x: ~(x.__current_object)
- __complex__ = lambda x: complex(x.__current_object)
- __int__ = lambda x: int(x.__current_object)
- __long__ = lambda x: long(x.__current_object)
- __float__ = lambda x: float(x.__current_object)
- __oct__ = lambda x: oct(x.__current_object)
- __hex__ = lambda x: hex(x.__current_object)
- __index__ = lambda x: x.__current_object.__index__()
- __coerce__ = lambda x, o: x.__coerce__(x, o)
- __enter__ = lambda x: x.__enter__()
- __exit__ = lambda x, *a, **kw: x.__exit__(*a, **kw)
-
-def to_sphinx(value):
- "Convert a value into a sphinx query value"
- if isinstance(value, date) or isinstance(value, datetime):
- return int(time.mktime(value.timetuple()))
- elif isinstance(value, decimal.Decimal) or isinstance(value, float):
- return float(value)
- return int(value)
-
-class SphinxQuerySet(object):
- available_kwargs = ('rankmode', 'mode', 'weights', 'maxmatches')
-
- def __init__(self, model=None, **kwargs):
- self._select_related = False
- self._select_related_args = {}
- self._select_related_fields = []
- self._filters = {}
- self._excludes = {}
- self._extra = {}
- self._query = ''
- self.__metadata = None
- self._offset = 0
- self._limit = 20
-
- self._groupby = None
- self._sort = None
- self._weights = [1, 100]
-
- self._maxmatches = 1000
- self._result_cache = None
- self._mode = sphinxapi.SPH_MATCH_ALL
- self._rankmode = getattr(sphinxapi, 'SPH_RANK_PROXIMITY_BM25', None)
- self._model = model
- self._anchor = {}
- self.__metadata = {}
-
- self.set_options(**kwargs)
-
- if model:
- self._index = kwargs.get('index', model._meta.db_table)
- else:
- self._index = kwargs.get('index')
-
- def __repr__(self):
- if self._result_cache is not None:
- return repr(self._get_data())
- else:
- return '<%s instance>' % (self.__class__.__name__,)
-
- def __len__(self):
- return len(self._get_data())
-
- def __iter__(self):
- return iter(self._get_data())
-
- def __getitem__(self, k):
- if not isinstance(k, (slice, int, long)):
- raise TypeError
- assert (not isinstance(k, slice) and (k >= 0)) \
- or (isinstance(k, slice) and (k.start is None or k.start >= 0) and (k.stop is None or k.stop >= 0)), \
- "Negative indexing is not supported."
- if type(k) == slice:
- if self._offset < k.start or k.stop-k.start > self._limit:
- self._result_cache = None
- else:
- if k not in range(self._offset, self._limit+self._offset):
- self._result_cache = None
- if self._result_cache is None:
- if type(k) == slice:
- self._offset = k.start
- self._limit = k.stop-k.start
- return self._get_results()
- else:
- self._offset = k
- self._limit = 1
- return self._get_results()[0]
- else:
- return self._result_cache[k]
-
- def set_options(self, **kwargs):
- if 'rankmode' in kwargs:
- if kwargs.get('rankmode') is None:
- kwargs['rankmode'] = sphinxapi.SPH_RANK_NONE
- for key in self.available_kwargs:
- if key in kwargs:
- setattr(self, '_%s' % (key,), kwargs[key])
-
- def query(self, string):
- return self._clone(_query=unicode(string).encode('utf-8'))
-
- def group_by(self, attribute, func, groupsort='@group desc'):
- return self._clone(_groupby=attribute, _groupfunc=func, _groupsort=groupsort)
-
- def rank_none(self):
- warnings.warn('`rank_none()` is deprecated. Use `set_options(rankmode=None)` instead.', DeprecationWarning)
- return self._clone(_rankmode=sphinxapi.SPH_RANK_NONE)
-
- def mode(self, mode):
- warnings.warn('`mode()` is deprecated. Use `set_options(mode='')` instead.', DeprecationWarning)
- return self._clone(_mode=mode)
-
- def weights(self, weights):
- warnings.warn('`mode()` is deprecated. Use `set_options(weights=[])` instead.', DeprecationWarning)
- return self._clone(_weights=weights)
-
- def on_index(self, index):
- warnings.warn('`mode()` is deprecated. Use `set_options(on_index=foo)` instead.', DeprecationWarning)
- return self._clone(_index=index)
-
- # only works on attributes
- def filter(self, **kwargs):
- filters = self._filters.copy()
- for k,v in kwargs.iteritems():
- if hasattr(v, 'next'):
- v = list(v)
- elif not (isinstance(v, list) or isinstance(v, tuple)):
- v = [v,]
- filters.setdefault(k, []).extend(map(to_sphinx, v))
- return self._clone(_filters=filters)
-
- def geoanchor(self, lat_attr, lng_attr, lat, lng):
- assert(sphinxapi.VER_COMMAND_SEARCH >= 0x113, "You must upgrade sphinxapi to version 0.98 to use Geo Anchoring.")
- return self._clone(_anchor=(lat_attr, lng_attr, float(lat), float(lng)))
-
- # this actually does nothing, its just a passthru to
- # keep things looking/working generally the same
- def all(self):
- return self
-
- # only works on attributes
- def exclude(self, **kwargs):
- filters = self._excludes.copy()
- for k,v in kwargs.iteritems():
- if hasattr(v, 'next'):
- v = list(v)
- elif not (isinstance(v, list) or isinstance(v, tuple)):
- v = [v,]
- filters.setdefault(k, []).extend(map(to_sphinx, v))
- return self._clone(_excludes=filters)
-
- # you cannot order by @weight (it always orders in descending)
- # keywords are @id, @weight, @rank, and @relevance
- def order_by(self, *args):
- sort_by = []
- for arg in args:
- sort = 'ASC'
- if arg[0] == '-':
- arg = arg[1:]
- sort = 'DESC'
- if arg == 'id':
- arg = '@id'
- sort_by.append('%s %s' % (arg, sort))
- if sort_by:
- return self._clone(_sort=(sphinxapi.SPH_SORT_EXTENDED, ', '.join(sort_by)))
- return self
-
- # pass these thru on the queryset and let django handle it
- def select_related(self, *args, **kwargs):
- _args = self._select_related_fields[:]
- _args.extend(args)
- _kwargs = self._select_related_args.copy()
- _kwargs.update(kwargs)
-
- return self._clone(
- _select_related=True,
- _select_related_fields=_args,
- _select_related_args=_kwargs,
- )
-
- def extra(self, **kwargs):
- extra = self._extra.copy()
- extra.update(kwargs)
- return self._clone(_extra=extra)
-
- def count(self):
- return min(self._sphinx.get('total_found', 0), self._maxmatches)
-
- def reset(self):
- return self.__class__(self._model, self._index)
-
- # Internal methods
- def _clone(self, **kwargs):
- # Clones the queryset passing any changed args
- c = self.__class__()
- c.__dict__.update(self.__dict__)
- c.__dict__.update(kwargs)
- return c
-
- def _sphinx(self):
- if not self.__metadata:
- # We have to force execution if this is accessed beforehand
- self._get_data()
- return self.__metadata
- _sphinx = property(_sphinx)
-
- def _get_data(self):
- assert(self._index)
- # need to find a way to make this work yet
- if self._result_cache is None:
- self._result_cache = list(self._get_results())
- return self._result_cache
-
- def _get_sphinx_results(self):
- assert(self._offset + self._limit <= self._maxmatches)
-
- client = sphinxapi.SphinxClient()
- client.SetServer(SPHINX_SERVER, SPHINX_PORT)
-
- if self._sort:
- client.SetSortMode(*self._sort)
-
- if isinstance(self._weights, dict):
- client.SetFieldWeights(self._weights)
- else:
- # assume its a list
- client.SetWeights(map(int, self._weights))
-
- client.SetMatchMode(self._mode)
-
- # 0.97 requires you to reset it
- if hasattr(client, 'ResetFilters'):
- client.ResetFilters()
- if hasattr(client, 'ResetGroupBy'):
- client.ResetGroupBy()
-
- def _handle_filters(filter_list, exclude=False):
- for name, values in filter_list.iteritems():
- parts = len(name.split('__'))
- if parts > 2:
- raise NotImplementedError, 'Related object and/or multiple field lookups not supported'
- elif parts == 2:
- # The float handling for __gt and __lt is kind of ugly..
- name, lookup = name.split('__', 1)
- is_float = isinstance(values[0], float)
- if lookup == 'gt':
- value = is_float and values[0] + (1.0/MAX_INT) or values[0] - 1
- args = (name, value, MAX_INT, exclude)
- elif lookup == 'gte':
- args = (name, values[0], MAX_INT, exclude)
- elif lookup == 'lt':
- value = is_float and values[0] - (1.0/MAX_INT) or values[0] - 1
- args = (name, -MAX_INT, value, exclude)
- elif lookup == 'lte':
- args = (name, -MAX_INT, values[0], exclude)
- elif lookup == 'range':
- args = (name, values[0], values[1], exclude)
- else:
- raise NotImplementedError, 'Related object and/or field lookup "%s" not supported' % lookup
- if is_float:
- client.SetFilterFloatRange(*args)
- elif not exclude and self._model and name == self._model._meta.pk.column:
- client.SetIDRange(*args[1:3])
- else:
- client.SetFilterRange(*args)
-
- else:
- client.SetFilter(name, values, exclude)
-
- # Include filters
- if self._filters:
- _handle_filters(self._filters)
-
- # Exclude filters
- if self._excludes:
- _handle_filters(self._excludes, True)
-
- if self._groupby:
- client.SetGroupBy(self._groupby, self._groupfunc, self._groupsort)
-
- if self._anchor:
- client.SetGeoAnchor(*self._anchor)
-
- if self._rankmode:
- client.SetRankingMode(self._rankmode)
-
- if not self._limit > 0:
- # Fix for Sphinx throwing an assertion error when you pass it an empty limiter
- return []
-
-
- if sphinxapi.VER_COMMAND_SEARCH >= 0x113:
- client.SetRetries(SPHINX_RETRIES, SPHINX_RETRIES_DELAY)
-
- client.SetLimits(int(self._offset), int(self._limit), int(self._maxmatches))
-
- results = client.Query(self._query, self._index)
-
- # The Sphinx API doesn't raise exceptions
- if not results:
- if client.GetLastError():
- raise SearchError, client.GetLastError()
- elif client.GetLastWarning():
- raise SearchError, client.GetLastWarning()
- return results
-
- def _get_results(self):
- results = self._get_sphinx_results()
- if not results or not results['matches']:
- results = []
- elif self._model:
- queryset = self._model.objects.all()
- if self._select_related:
- queryset = queryset.select_related(*self._select_related_fields, **self._select_related_args)
- if self._extra:
- queryset = queryset.extra(**self._extra)
- pks = getattr(self._model._meta, 'pks', None)
- if pks is None or len(pks) == 1:
- queryset = queryset.filter(pk__in=[r['id'] for r in results['matches']])
- queryset = dict([(o.pk, o) for o in queryset])
- else:
- for r in results['matches']:
- r['id'] = ', '.join([unicode(r['attrs'][p.column]) for p in pks])
- q = reduce(operator.or_, [reduce(operator.and_, [Q(**{p.name: r['attrs'][p.column]}) for p in pks]) for r in results['matches']])
- if q:
- queryset = queryset.filter(q)
- queryset = dict([(', '.join([unicode(p) for p in o.pks]), o) for o in queryset])
- else:
- queryset = None
-
- if queryset:
- self.__metadata = {
- 'total': results['total'],
- 'total_found': results['total_found'],
- 'words': results['words'],
- }
- results = [SphinxProxy(queryset[r['id']], r) for r in results['matches'] if r['id'] in queryset]
- else:
- results = []
- else:
- "We did a query without a model, lets see if there's a content_type"
- results['attrs'] = dict(results['attrs'])
- if 'content_type' in results['attrs']:
- "Now we have to do one query per content_type"
- objcache = {}
- for r in results['matches']:
- ct = r['attrs']['content_type']
- if ct not in objcache:
- objcache[ct] = {}
- objcache[ct][r['id']] = None
- for ct in objcache:
- queryset = ContentType.objects.get(pk=ct).model_class().objects.filter(pk__in=objcache[ct])
- for o in queryset:
- objcache[ct][o.id] = o
- results = [objcache[r['attrs']['content_type']][r['id']] for r in results['matches']]
- else:
- results = results['matches']
- self._result_cache = results
- return results
-
-class SphinxModelManager(object):
- def __init__(self, model, **kwargs):
- self._model = model
- self._index = kwargs.pop('index', model._meta.db_table)
- self._kwargs = kwargs
-
- def _get_query_set(self):
- return SphinxQuerySet(self._model, index=self._index, **self._kwargs)
-
- def get_index(self):
- return self._index
-
- def all(self):
- return self._get_query_set()
-
- def filter(self, **kwargs):
- return self._get_query_set().filter(**kwargs)
-
- def query(self, *args, **kwargs):
- return self._get_query_set().query(*args, **kwargs)
-
- def on_index(self, *args, **kwargs):
- return self._get_query_set().on_index(*args, **kwargs)
-
- def geoanchor(self, *args, **kwargs):
- return self._get_query_set().geoanchor(*args, **kwargs)
-
-class SphinxInstanceManager(object):
- """Collection of tools useful for objects which are in a Sphinx index."""
- def __init__(self, instance, index):
- self._instance = instance
- self._index = index
-
- def update(self, **kwargs):
- assert(sphinxapi.VER_COMMAND_SEARCH >= 0x113, "You must upgrade sphinxapi to version 0.98 to use Geo Anchoring.")
- sphinxapi.UpdateAttributes(index, kwargs.keys(), dict(self.instance.pk, map(to_sphinx, kwargs.values())))
-
-
-class SphinxSearch(object):
- def __init__(self, index=None, **kwargs):
- self._kwargs = kwargs
- self._sphinx = None
- self._index = index
- self.model = None
-
- def __call__(self, index, **kwargs):
- warnings.warn('For non-model searches use a SphinxQuerySet instance.', DeprecationWarning)
- return SphinxQuerySet(index=index, **kwargs)
-
- def __get__(self, instance, model, **kwargs):
- if instance:
- return SphinxInstanceManager(instance, index)
- return self._sphinx
-
- def contribute_to_class(self, model, name, **kwargs):
- if self._index is None:
- self._index = model._meta.db_table
- self._sphinx = SphinxModelManager(model, index=self._index, **self._kwargs)
- self.model = model
- if getattr(model, '__sphinx_indexes__', None) is None:
- setattr(model, '__sphinx_indexes__', [self._index])
- else:
- model.__sphinx_indexes__.append(self._index)
- setattr(model, name, self._sphinx)
-
-class SphinxRelationProxy(SphinxProxy):
- def count(self):
- return min(self._sphinx['attrs']['@count'], self._maxmatches)
-
-class SphinxRelation(SphinxSearch):
- """
- Adds "related model" support to django-sphinx --
- http://code.google.com/p/django-sphinx/
- http://www.sphinxsearch.com/
-
- Example --
-
- class MySearch(SphinxSearch):
- myrelatedobject = SphinxRelation(RelatedModel)
- anotherone = SphinxRelation(AnotherModel)
- ...
-
- class MyModel(models.Model):
- search = MySearch('index')
-
- """
- def __init__(self, model=None, attr=None, sort='@count desc', **kwargs):
- if model:
- self._related_model = model
- self._related_attr = attr or model.__name__.lower()
- self._related_sort = sort
- super(SphinxRelation, self).__init__(**kwargs)
-
- def __get__(self, instance, instance_model, **kwargs):
- self._mode = instance._mode
- self._rankmode = instance._rankmode
- self._index = instance._index
- self._query = instance._query
- self._filters = instance._filters
- self._excludes = instance._excludes
- self._model = self._related_model
- self._groupby = self._related_attr
- self._groupsort = self._related_sort
- self._groupfunc = sphinxapi.SPH_GROUPBY_ATTR
- return self
-
- def _get_results(self):
- results = self._get_sphinx_results()
- if not results: return []
- if results['matches'] and self._model:
- ids = []
- for r in results['matches']:
- value = r['attrs']['@groupby']
- if isinstance(value, (int, long)):
- ids.append(value)
- else:
- ids.extend()
- qs = self._model.objects.filter(pk__in=set(ids))
- if self._select_related:
- qs = qs.select_related(*self._select_related_fields,
- **self._select_related_args)
- if self._extra:
- qs = qs.extra(**self._extra)
- queryset = dict([(o.id, o) for o in qs])
- self.__metadata = {
- 'total': results['total'],
- 'total_found': results['total_found'],
- 'words': results['words'],
- }
- results = [ SphinxRelationProxy(queryset[k['attrs']['@groupby']], k) \
- for k in results['matches'] \
- if k['attrs']['@groupby'] in queryset ]
- else:
- results = []
- self._result_cache = results
- return results
-
- def _sphinx(self):
- if not self.__metadata:
- # We have to force execution if this is accessed beforehand
- self._get_data()
- return self.__metadata
- _sphinx = property(_sphinx)
\ No newline at end of file
fnp / wolnelektury.git / blobdiff