Merge remote-tracking branch 'upstream/master'

Author: Kenneth Yu

.travis.yml

@@ -5,6 +5,7 @@ python:
     - 2.7
     - 3.4
     - 3.5
+    - 3.6
     - pypy
 
 cache:
@@ -58,17 +59,19 @@ script:
 
 env:
     matrix:
-        - DJANGO_VERSION=">=1.8,<1.9" VERSION_ES=">=1.0.0,<2.0.0"
-        - DJANGO_VERSION=">=1.9,<1.10" VERSION_ES=">=1.0.0,<2.0.0"
-        - DJANGO_VERSION=">=1.10,<1.11" VERSION_ES=">=1.0.0,<2.0.0"
-        - DJANGO_VERSION=">=1.8,<1.9" VERSION_ES=">=2.0.0,<3.0.0"
-        - DJANGO_VERSION=">=1.9,<1.10" VERSION_ES=">=2.0.0,<3.0.0"
-        - DJANGO_VERSION=">=1.10,<1.11" VERSION_ES=">=2.0.0,<3.0.0"
-        - DJANGO_VERSION=">=1.11,<1.12" VERSION_ES=">=2.0.0,<3.0.0"
+        - DJANGO_VERSION=">=1.11,<2.0" VERSION_ES=">=1.0.0,<2.0.0"
+        - DJANGO_VERSION=">=2.0,<2.1" VERSION_ES=">=1.0.0,<2.0.0"
+        - DJANGO_VERSION=">=1.11,<2.0" VERSION_ES=">=2.0.0,<3.0.0"
+        - DJANGO_VERSION=">=2.0,<2.1" VERSION_ES=">=2.0.0,<3.0.0"
 
 matrix:
     allow_failures:
         - python: 'pypy'
+    exclude:
+        - python: 2.7
+          env: DJANGO_VERSION=">=2.0,<2.1" VERSION_ES=">=2.0.0,<3.0.0"
+        - python: 2.7
+          env: DJANGO_VERSION=">=2.0,<2.1" VERSION_ES=">=1.0.0,<2.0.0"
 
 notifications:
     irc: "irc.freenode.org#haystack"

AUTHORS

@@ -117,3 +117,4 @@ Thanks to
     * Morgan Aubert (@ellmetha) for Django 1.10 support
     * João Junior (@joaojunior) and Bruno Marques (@ElSaico) for Elasticsearch 2.x support
     * Alex Tomkins (@tomkins) for various patches
+    * Martin Pauly (@mpauly) for Django 2.0 support

haystack/__init__.py

@@ -4,7 +4,7 @@
 
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
-from pkg_resources import DistributionNotFound, get_distribution
+from pkg_resources import DistributionNotFound, get_distribution, parse_version
 
 from haystack import signals
 from haystack.constants import DEFAULT_ALIAS
@@ -13,9 +13,12 @@
 __author__ = 'Daniel Lindsley'
 
 try:
-    __version__ = get_distribution(__name__).version
+    pkg_distribution = get_distribution(__name__)
+    __version__ = pkg_distribution.version
+    version_info = pkg_distribution.parsed_version
 except DistributionNotFound:
-    __version__ = (0, 0, 'dev0')
+    __version__ = '0.0.dev0'
+    version_info = parse_version(__version__)
 
 default_app_config = 'haystack.apps.HaystackConfig'
 

haystack/backends/elasticsearch_backend.py

@@ -296,7 +296,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
             for field, direction in sort_by:
                 if field == 'distance' and distance_point:
                     # Do the geo-enabled sort.
-                    lng, lat = distance_point['point'].get_coords()
+                    lng, lat = distance_point['point'].coords
                     sort_kwargs = {
                         "_geo_distance": {
                             distance_point['field']: [lng, lat],
@@ -455,7 +455,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
             filters.append(within_filter)
 
         if dwithin is not None:
-            lng, lat = dwithin['point'].get_coords()
+            lng, lat = dwithin['point'].coords
 
             # NB: the 1.0.0 release of elasticsearch introduce an
             #     incompatible change on the distance filter formating
@@ -625,8 +625,8 @@ def from_timestamp(tm):
             model = haystack_get_model(app_label, model_name)
 
             if model and model in indexed_models:
+                index = source and unified_index.get_index(model)
                 for key, value in source.items():
-                    index = unified_index.get_index(model)
                     string_key = str(key)
 
                     if string_key in index.fields and hasattr(index.fields[string_key], 'convert'):

haystack/backends/solr_backend.py

@@ -173,7 +173,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
         if sort_by is not None:
             if sort_by in ['distance asc', 'distance desc'] and distance_point:
                 # Do the geo-enabled sort.
-                lng, lat = distance_point['point'].get_coords()
+                lng, lat = distance_point['point'].coords
                 kwargs['sfield'] = distance_point['field']
                 kwargs['pt'] = '%s,%s' % (lat, lng)
 
@@ -290,7 +290,7 @@ def build_search_kwargs(self, query_string, sort_by=None, start_offset=0, end_of
 
         if dwithin is not None:
             kwargs.setdefault('fq', [])
-            lng, lat = dwithin['point'].get_coords()
+            lng, lat = dwithin['point'].coords
             geofilt = '{!geofilt pt=%s,%s sfield=%s d=%s}' % (lat, lng, dwithin['field'], dwithin['distance'].km)
             kwargs['fq'].append(geofilt)
 

haystack/backends/whoosh_backend.py

@@ -277,7 +277,7 @@ def optimize(self):
     def calculate_page(self, start_offset=0, end_offset=None):
         # Prevent against Whoosh throwing an error. Requires an end_offset
         # greater than 0.
-        if not end_offset is None and end_offset <= 0:
+        if end_offset is not None and end_offset <= 0:
             end_offset = 1
 
         # Determine the page.
@@ -353,7 +353,7 @@ def search(self, query_string, sort_by=None, start_offset=0, end_offset=None,
                     if len(sort_by_list) == 1:
                         reverse = False
 
-            sort_by = sort_by_list[0]
+            sort_by = sort_by_list
 
         if facets is not None:
             warnings.warn("Whoosh does not handle faceting.", Warning, stacklevel=2)
@@ -404,7 +404,7 @@ def search(self, query_string, sort_by=None, start_offset=0, end_offset=None,
                 if narrowed_results:
                     narrowed_results.filter(recent_narrowed_results)
                 else:
-                   narrowed_results = recent_narrowed_results
+                    narrowed_results = recent_narrowed_results
 
         self.index = self.index.refresh()
 
@@ -484,10 +484,6 @@ def more_like_this(self, model_instance, additional_query_string=None,
         if not self.setup_complete:
             self.setup()
 
-        # Deferred models will have a different class ("RealClass_Deferred_fieldname")
-        # which won't be in our registry:
-        model_klass = model_instance._meta.concrete_model
-
         field_name = self.content_field_name
         narrow_queries = set()
         narrowed_results = None
@@ -533,13 +529,14 @@ def more_like_this(self, model_instance, additional_query_string=None,
                 if narrowed_results:
                     narrowed_results.filter(recent_narrowed_results)
                 else:
-                   narrowed_results = recent_narrowed_results
+                    narrowed_results = recent_narrowed_results
 
         page_num, page_length = self.calculate_page(start_offset, end_offset)
 
         self.index = self.index.refresh()
         raw_results = EmptyResults()
 
+        searcher = None
         if self.index.doc_count():
             query = "%s:%s" % (ID, get_identifier(model_instance))
             searcher = self.index.searcher()
@@ -575,7 +572,9 @@ def more_like_this(self, model_instance, additional_query_string=None,
             }
 
         results = self._process_results(raw_page, result_class=result_class)
-        searcher.close()
+
+        if searcher:
+            searcher.close()
 
         if hasattr(narrow_searcher, 'close'):
             narrow_searcher.close()
@@ -900,13 +899,6 @@ def build_query_fragment(self, field, filter_type, value):
         return u"%s%s" % (index_fieldname, query_frag)
 
 
-        # if not filter_type in ('in', 'range'):
-        #     # 'in' is a bit of a special case, as we don't want to
-        #     # convert a valid list/tuple to string. Defer handling it
-        #     # until later...
-        #     value = self.backend._from_python(value)
-
-
 class WhooshEngine(BaseEngine):
     backend = WhooshSearchBackend
     query = WhooshSearchQuery

haystack/fields.py

@@ -3,7 +3,7 @@
 
 import re
 
-from django.template import Context, loader
+from django.template import loader
 from django.utils import datetime_safe, six
 
 from haystack.exceptions import SearchFieldError
@@ -112,7 +112,8 @@ def resolve_attributes_lookup(self, current_objects, attributes):
 
             if len(attributes) > 1:
                 current_objects_in_attr = self.get_iterable_objects(getattr(current_object, attributes[0]))
-                return self.resolve_attributes_lookup(current_objects_in_attr, attributes[1:])
+                values.extend(self.resolve_attributes_lookup(current_objects_in_attr, attributes[1:]))
+                continue
 
             current_object = getattr(current_object, attributes[0])
 
@@ -244,6 +245,8 @@ def convert(self, value):
         elif isinstance(value, dict):
             lat = value.get('lat', 0)
             lng = value.get('lon', 0)
+        else:
+            raise TypeError('Unable to extract coordinates from %r' % value)
 
         value = Point(float(lng), float(lat))
         return value

haystack/generic_views.py

@@ -58,7 +58,7 @@ def get_form_kwargs(self):
         Returns the keyword arguments for instantiating the form.
         """
         kwargs = {'initial': self.get_initial()}
-        if self.request.method == 'GET' and self.request.GET:
+        if self.request.method == 'GET':
             kwargs.update({
                 'data': self.request.GET,
             })

haystack/indexes.py

@@ -443,7 +443,7 @@ def should_skip_field(self, field):
             return True
 
         # Ignore certain fields (AutoField, related fields).
-        if field.primary_key or getattr(field, 'rel'):
+        if field.primary_key or field.is_relation:
             return True
 
         return False

haystack/inputs.py

@@ -21,7 +21,7 @@ def __init__(self, query_string, **kwargs):
         self.kwargs = kwargs
 
     def __repr__(self):
-        return u"<%s '%s'>" % (self.__class__.__name__, self.__unicode__().encode('utf8'))
+        return u"<%s '%s'>" % (self.__class__.__name__, self)
 
     def __str__(self):
         return force_text(self.query_string)

haystack/management/commands/rebuild_index.py

@@ -1,5 +1,4 @@
 # encoding: utf-8
-
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 from django.core.management import call_command
@@ -33,5 +32,11 @@ def add_arguments(self, parser):
         )
 
     def handle(self, **options):
-        call_command('clear_index', **options)
-        call_command('update_index', **options)
+        clear_options = options.copy()
+        update_options = options.copy()
+        for key in ('batchsize', 'workers'):
+            del clear_options[key]
+        for key in ('interactive', ):
+            del update_options[key]
+        call_command('clear_index', **clear_options)
+        call_command('update_index', **update_options)

haystack/management/commands/update_index.py

@@ -131,6 +131,7 @@ def do_update(backend, index, qs, start, end, total, verbosity=1, commit=True,
     reset_queries()
     return max_pk
 
+
 class Command(BaseCommand):
     help = "Freshens the index for the given app(s)."
 
@@ -145,11 +146,13 @@ def add_arguments(self, parser):
         )
         parser.add_argument(
             '-s', '--start', dest='start_date',
-            help='The start date for indexing within. Can be any dateutil-parsable string, recommended to be YYYY-MM-DDTHH:MM:SS.'
+            help='The start date for indexing. Can be any dateutil-parsable string;'
+                 ' YYYY-MM-DDTHH:MM:SS is recommended to avoid confusion'
         )
         parser.add_argument(
             '-e', '--end', dest='end_date',
-            help='The end date for indexing within. Can be any dateutil-parsable string, recommended to be YYYY-MM-DDTHH:MM:SS.'
+            help='The end date for indexing. Can be any dateutil-parsable string;'
+                 ' YYYY-MM-DDTHH:MM:SS is recommended to avoid confusion'
         )
         parser.add_argument(
             '-b', '--batch-size', dest='batchsize', type=int,
@@ -268,9 +271,9 @@ def update_backend(self, label, using):
 
                 if self.workers == 0:
                     max_pk = do_update(backend, index, qs, start, end, total,
-                        verbosity=self.verbosity,
-                        commit=self.commit, max_retries=self.max_retries,
-                        last_max_pk=max_pk)
+                                       verbosity=self.verbosity,
+                                       commit=self.commit, max_retries=self.max_retries,
+                                       last_max_pk=max_pk)
                 else:
                     ghetto_queue.append((model, start, end, total, using, self.start_date, self.end_date,
                                          self.verbosity, self.commit, self.max_retries))
@@ -296,8 +299,6 @@ def update_backend(self, label, using):
                     # all pks. Rebuild the list with everything.
                     qs = index.index_queryset().values_list('pk', flat=True)
                     database_pks = set(smart_bytes(pk) for pk in qs)
-
-                    total = len(database_pks)
                 else:
                     database_pks = set(smart_bytes(pk) for pk in qs.values_list('pk', flat=True))
 

haystack/models.py

@@ -133,7 +133,7 @@ def _get_distance(self):
             if location_field is None:
                 return None
 
-            lf_lng, lf_lat = location_field.get_coords()
+            lf_lng, lf_lat = location_field.coords
             self._distance = Distance(km=geopy_distance.distance((po_lat, po_lng), (lf_lat, lf_lng)).km)
 
         # We've either already calculated it or the backend returned it, so

haystack/query.py

@@ -147,12 +147,12 @@ def _manual_iter(self):
                 current_position += 1
 
             if self._cache_is_full():
-                raise StopIteration
+                return
 
             # We've run out of results and haven't hit our limit.
             # Fill more of the cache.
             if not self._fill_cache(current_position, current_position + ITERATOR_LOAD_PER_QUERY):
-                raise StopIteration
+                return
 
     def post_process_results(self, results):
         to_cache = []
@@ -172,23 +172,32 @@ def post_process_results(self, results):
 
         for result in results:
             if self._load_all:
-                # We have to deal with integer keys being cast from strings
+
                 model_objects = loaded_objects.get(result.model, {})
-                if result.pk not in model_objects:
+                # Try to coerce a primary key object that matches the models pk
+                # We have to deal with semi-arbitrary keys being cast from strings (UUID, int, etc)
+                if model_objects:
+                    result_klass = type(next(iter(model_objects)))
+                    result.pk = result_klass(result.pk)
+
                     try:
-                        result.pk = int(result.pk)
-                    except ValueError:
-                        pass
-                try:
-                    result._object = model_objects[result.pk]
-                except KeyError:
-                    # The object was either deleted since we indexed or should
-                    # be ignored; fail silently.
-                    self._ignored_result_count += 1
-
-                    # avoid an unfilled None at the end of the result cache
-                    self._result_cache.pop()
-                    continue
+                        result._object = model_objects[result.pk]
+                    except KeyError:
+                        # The object was either deleted since we indexed or should
+                        # be ignored for other reasons such as an overriden 'load_all_queryset';
+                        # fail silently.
+                        self._ignored_result_count += 1
+
+                        # avoid an unfilled None at the end of the result cache
+                        self._result_cache.pop()
+                        continue
+                else:
+                    # No objects were returned -- possible due to SQS nesting such as
+                    # XYZ.objects.filter(id__gt=10) where the amount ignored are
+                    # exactly equal to the ITERATOR_LOAD_PER_QUERY
+                    del self._result_cache[:len(results)]
+                    self._ignored_result_count += len(results)
+                    break
 
             to_cache.append(result)