Allow individual records to be skipped while indexing

Previously there was no easy way to skip specific objects other than
filtering the queryset. This change allows a prepare method to raise
`SkipDocument` after calling methods or making other checks which cannot
easily be expressed as database filters.

Thanks to Felipe Prenholato (@chronossc) for the patch

Closes django-haystack#380
Closes django-haystack#1191

Author: acdha

AUTHORS

@@ -99,3 +99,4 @@ Thanks to
     * Andrei Fokau (andreif) for adding support for ``SQ`` in ``SearchQuerySet.narrow()``
     * Phill Tornroth (phill-tornroth) for several patches improving UnifiedIndex and ElasticSearch support
     * Philippe Luickx (philippeluickx) for documenting how to provide backend-specific facet options
+    * Felipe Prenholato (@chronossc) for a patch making it easy to exclude documents from indexing using custom logic

haystack/backends/elasticsearch_backend.py

@@ -14,7 +14,7 @@
 import haystack
 from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, log_query
 from haystack.constants import DEFAULT_OPERATOR, DJANGO_CT, DJANGO_ID, ID
-from haystack.exceptions import MissingDependency, MoreLikeThisError
+from haystack.exceptions import MissingDependency, MoreLikeThisError, SkipDocument
 from haystack.inputs import Clean, Exact, PythonData, Raw
 from haystack.models import SearchResult
 from haystack.utils import log as logging
@@ -172,6 +172,8 @@ def update(self, index, iterable, commit=True):
                 final_data['_id'] = final_data[ID]
 
                 prepped_docs.append(final_data)
+            except SkipDocument:
+                self.log.debug(u"Indexing for object `%s` skipped", obj)
             except elasticsearch.TransportError as e:
                 if not self.silently_fail:
                     raise

haystack/backends/solr_backend.py

@@ -11,7 +11,7 @@
 
 from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, EmptyResults, log_query
 from haystack.constants import DJANGO_CT, DJANGO_ID, ID
-from haystack.exceptions import MissingDependency, MoreLikeThisError
+from haystack.exceptions import MissingDependency, MoreLikeThisError, SkipDocument
 from haystack.inputs import Clean, Exact, PythonData, Raw
 from haystack.models import SearchResult
 from haystack.utils import log as logging
@@ -54,6 +54,8 @@ def update(self, index, iterable, commit=True):
         for obj in iterable:
             try:
                 docs.append(index.full_prepare(obj))
+            except SkipDocument:
+                self.log.debug(u"Indexing for object `%s` skipped", obj)
             except UnicodeDecodeError:
                 if not self.silently_fail:
                     raise

haystack/backends/whoosh_backend.py

@@ -16,7 +16,7 @@
 
 from haystack.backends import BaseEngine, BaseSearchBackend, BaseSearchQuery, EmptyResults, log_query
 from haystack.constants import DJANGO_CT, DJANGO_ID, ID
-from haystack.exceptions import MissingDependency, SearchBackendError
+from haystack.exceptions import MissingDependency, SearchBackendError, SkipDocument
 from haystack.inputs import Clean, Exact, PythonData, Raw
 from haystack.models import SearchResult
 from haystack.utils import log as logging
@@ -192,32 +192,35 @@ def update(self, index, iterable, commit=True):
         writer = AsyncWriter(self.index)
 
         for obj in iterable:
-            doc = index.full_prepare(obj)
-
-            # Really make sure it's unicode, because Whoosh won't have it any
-            # other way.
-            for key in doc:
-                doc[key] = self._from_python(doc[key])
-
-            # Document boosts aren't supported in Whoosh 2.5.0+.
-            if 'boost' in doc:
-                del doc['boost']
-
             try:
-                writer.update_document(**doc)
-            except Exception as e:
-                if not self.silently_fail:
-                    raise
-
-                # We'll log the object identifier but won't include the actual object
-                # to avoid the possibility of that generating encoding errors while
-                # processing the log message:
-                self.log.error(u"%s while preparing object for update" % e.__class__.__name__, exc_info=True, extra={
-                    "data": {
-                        "index": index,
-                        "object": get_identifier(obj)
-                    }
-                })
+                doc = index.full_prepare(obj)
+            except SkipDocument:
+                self.log.debug(u"Indexing for object `%s` skipped", obj)
+            else:
+                # Really make sure it's unicode, because Whoosh won't have it any
+                # other way.
+                for key in doc:
+                    doc[key] = self._from_python(doc[key])
+
+                # Document boosts aren't supported in Whoosh 2.5.0+.
+                if 'boost' in doc:
+                    del doc['boost']
+
+                try:
+                    writer.update_document(**doc)
+                except Exception as e:
+                    if not self.silently_fail:
+                        raise
+
+                    # We'll log the object identifier but won't include the actual object
+                    # to avoid the possibility of that generating encoding errors while
+                    # processing the log message:
+                    self.log.error(u"%s while preparing object for update" % e.__class__.__name__, exc_info=True, extra={
+                        "data": {
+                            "index": index,
+                            "object": get_identifier(obj)
+                        }
+                    })
 
         if len(iterable) > 0:
             # For now, commit no matter what, as we run into locking issues otherwise.

haystack/exceptions.py

@@ -7,34 +7,47 @@ class HaystackError(Exception):
     """A generic exception for all others to extend."""
     pass
 
+
 class SearchBackendError(HaystackError):
     """Raised when a backend can not be found."""
     pass
 
+
 class SearchFieldError(HaystackError):
     """Raised when a field encounters an error."""
     pass
 
+
 class MissingDependency(HaystackError):
     """Raised when a library a backend depends on can not be found."""
     pass
 
+
 class NotHandled(HaystackError):
     """Raised when a model is not handled by the router setup."""
     pass
 
+
 class MoreLikeThisError(HaystackError):
     """Raised when a model instance has not been provided for More Like This."""
     pass
 
+
 class FacetingError(HaystackError):
     """Raised when incorrect arguments have been provided for faceting."""
     pass
 
+
 class SpatialError(HaystackError):
     """Raised when incorrect arguments have been provided for spatial."""
     pass
 
+
 class StatsError(HaystackError):
     "Raised when incorrect arguments have been provided for stats"
     pass
+
+
+class SkipDocument(HaystackError):
+    """Raised when a document should be skipped while updating"""
+    pass

test_haystack/elasticsearch_tests/test_elasticsearch_backend.py

@@ -13,6 +13,7 @@
 from django.utils import unittest
 
 from haystack import connections, indexes, reset_search_queries
+from haystack.exceptions import SkipDocument
 from haystack.inputs import AutoQuery
 from haystack.models import SearchResult
 from haystack.query import RelatedSearchQuerySet, SearchQuerySet, SQ
@@ -57,6 +58,14 @@ def get_model(self):
         return MockModel
 
 
+class ElasticsearchMockSearchIndexWithSkipDocument(ElasticsearchMockSearchIndex):
+
+    def prepare_text(self, obj):
+        if obj.author == 'daniel3':
+            raise SkipDocument
+        return u"Indexed!\n%s" % obj.id
+
+
 class ElasticsearchMockSpellingIndex(indexes.SearchIndex, indexes.Indexable):
     text = indexes.CharField(document=True)
     name = indexes.CharField(model_attr='author', faceted=True)
@@ -207,6 +216,7 @@ def get_model(self):
 
 
 class TestSettings(TestCase):
+
     def test_kwargs_are_passed_on(self):
         from haystack.backends.elasticsearch_backend import ElasticsearchSearchBackend
         backend = ElasticsearchSearchBackend('alias', **{
@@ -230,6 +240,7 @@ def setUp(self):
         self.old_ui = connections['elasticsearch'].get_unified_index()
         self.ui = UnifiedIndex()
         self.smmi = ElasticsearchMockSearchIndex()
+        self.smmidni = ElasticsearchMockSearchIndexWithSkipDocument()
         self.smtmmi = ElasticsearchMaintainTypeMockSearchIndex()
         self.ui.build(indexes=[self.smmi])
         connections['elasticsearch']._index = self.ui
@@ -335,6 +346,18 @@ def test_update(self):
             }
         ])
 
+    def test_update_with_SkipDocument_raised(self):
+        self.sb.update(self.smmidni, self.sample_objs)
+
+        # Check what Elasticsearch thinks is there.
+        res = self.raw_search('*:*')['hits']
+        self.assertEqual(res['total'], 2)
+        self.assertListEqual(
+            sorted([x['_source']['id'] for x in res['hits']]),
+            ['core.mockmodel.1', 'core.mockmodel.2']
+        )
+
+
     def test_remove(self):
         self.sb.update(self.smmi, self.sample_objs)
         self.assertEqual(self.raw_search('*:*')['hits']['total'], 3)

test_haystack/solr_tests/test_solr_backend.py

@@ -14,6 +14,7 @@
 from mock import patch
 
 from haystack import connections, indexes, reset_search_queries
+from haystack.exceptions import SkipDocument
 from haystack.inputs import AltParser, AutoQuery, Raw
 from haystack.models import SearchResult
 from haystack.query import RelatedSearchQuerySet, SearchQuerySet, SQ
@@ -50,6 +51,14 @@ def get_model(self):
         return MockModel
 
 
+class SolrMockSearchIndexWithSkipDocument(SolrMockSearchIndex):
+
+        def prepare_text(self, obj):
+            if obj.author == 'daniel3':
+                raise SkipDocument
+            return u"Indexed!\n%s" % obj.id
+
+
 class SolrMockOverriddenFieldNameSearchIndex(indexes.SearchIndex, indexes.Indexable):
     text = indexes.CharField(document=True, use_template=True)
     name = indexes.CharField(model_attr='author', faceted=True, index_fieldname='name_s')
@@ -200,6 +209,7 @@ def setUp(self):
         self.old_ui = connections['solr'].get_unified_index()
         self.ui = UnifiedIndex()
         self.smmi = SolrMockSearchIndex()
+        self.smmidni = SolrMockSearchIndexWithSkipDocument()
         self.smtmmi = SolrMaintainTypeMockSearchIndex()
         self.smofnmi = SolrMockOverriddenFieldNameSearchIndex()
         self.ui.build(indexes=[self.smmi])
@@ -285,6 +295,18 @@ def test_update(self):
             }
         ])
 
+    def test_update_with_SkipDocument_raised(self):
+        self.sb.update(self.smmidni, self.sample_objs)
+
+        res = self.raw_solr.search('*:*')
+
+        # Check what Solr thinks is there.
+        self.assertEqual(res.hits, 2)
+        self.assertListEqual(
+            sorted([x['id'] for x in res.docs]),
+            ['core.mockmodel.1', 'core.mockmodel.2']
+        )
+
     def test_remove(self):
         self.sb.update(self.smmi, self.sample_objs)
         self.assertEqual(self.raw_solr.search('*:*').hits, 3)

test_haystack/whoosh_tests/test_whoosh_backend.py

@@ -15,7 +15,7 @@
 from whoosh.qparser import QueryParser
 
 from haystack import connections, indexes, reset_search_queries
-from haystack.exceptions import SearchBackendError
+from haystack.exceptions import SkipDocument, SearchBackendError
 from haystack.inputs import AutoQuery
 from haystack.models import SearchResult
 from haystack.query import SearchQuerySet, SQ
@@ -35,6 +35,14 @@ def get_model(self):
         return MockModel
 
 
+class WhooshMockSearchIndexWithSkipDocument(WhooshMockSearchIndex):
+
+    def prepare_text(self, obj):
+        if obj.author == 'daniel3':
+            raise SkipDocument
+        return obj.author
+
+
 class WhooshAnotherMockSearchIndex(indexes.SearchIndex, indexes.Indexable):
     text = indexes.CharField(document=True)
     name = indexes.CharField(model_attr='author')
@@ -115,6 +123,7 @@ def setUp(self):
         self.old_ui = connections['whoosh'].get_unified_index()
         self.ui = UnifiedIndex()
         self.wmmi = WhooshMockSearchIndex()
+        self.wmmidni = WhooshMockSearchIndexWithSkipDocument()
         self.wmtmmi = WhooshMaintainTypeMockSearchIndex()
         self.ui.build(indexes=[self.wmmi])
         self.sb = connections['whoosh'].get_backend()
@@ -172,6 +181,18 @@ def test_update(self):
         self.assertEqual(len(self.whoosh_search(u'*')), 23)
         self.assertEqual([doc.fields()['id'] for doc in self.whoosh_search(u'*')], [u'core.mockmodel.%s' % i for i in range(1, 24)])
 
+    def test_update_with_SkipDocument_raised(self):
+        self.sb.update(self.wmmidni, self.sample_objs)
+
+        # Check what Whoosh thinks is there.
+        res = self.whoosh_search(u'*')
+        self.assertEqual(len(res), 14)
+        ids = [1, 2, 5, 6, 7, 8, 9, 11, 12, 14, 15, 18, 20, 21]
+        self.assertListEqual(
+            [doc.fields()['id'] for doc in res],
+            [u'core.mockmodel.%s' % i for i in ids]
+        )
+
     def test_remove(self):
         self.sb.update(self.wmmi, self.sample_objs)
         self.assertEqual(self.sb.index.doc_count(), 23)