new: allow Routers to return multiple indexes

Thanks to Hugo Chargois (@hchargois) for the patch

Closes django-haystack#1337
Closes django-haystack#934

Author: acdha

docs/multiple_index.rst

@@ -69,7 +69,15 @@ Automatic Routing
 
 To make the selection of the correct index easier, Haystack (like Django) has
 the concept of "routers". All provided routers are checked whenever a read or
-write happens, stopping at the first router that knows how to handle it.
+write happens, in the order in which they are defined.
+
+For read operations (when a search query is executed), the ``for_read`` method
+of each router is called, until one of them returns an index, which is used for
+the read operation.
+
+For write operations (when a delete or update is executed), the ``for_write``
+method of each router is called, and the results are aggregated. All of the
+indexes that were returned are then updated.
 
 Haystack ships with a ``DefaultRouter`` enabled. It looks like::
 
@@ -80,13 +88,18 @@ Haystack ships with a ``DefaultRouter`` enabled. It looks like::
         def for_write(self, **hints):
             return DEFAULT_ALIAS
 
-On a read (when a search query is executed), the ``DefaultRouter.for_read``
-method is checked & returns the ``DEFAULT_ALIAS`` (which is ``default``),
-telling whatever requested it that it should perform the query against the
-``default`` connection. The same process is followed for writes.
+This means that the default index is used for all read and write operations.
+
+If the ``for_read`` or ``for_write`` method doesn't exist or returns ``None``,
+that indicates that the current router can't handle the data. The next router
+is then checked.
 
-If the ``for_read`` or ``for_write`` method returns ``None``, that indicates
-that the current router can't handle the data. The next router is then checked.
+The ``for_write`` method can return either a single string representing an
+index name, or an iterable of such index names. For example::
+
+    class UpdateEverythingRouter(BaseRouter):
+        def for_write(self, **hints):
+            return ('myindex1', 'myindex2')
 
 The ``hints`` passed can be anything that helps the router make a decision. This
 data should always be considered optional & be guarded against. At current,
@@ -99,7 +112,6 @@ You may provide as many routers as you like by overriding the
 
     HAYSTACK_ROUTERS = ['myapp.routers.MasterRouter', 'myapp.routers.SlaveRouter', 'haystack.routers.DefaultRouter']
 
-
 Master-Slave Example
 --------------------
 

haystack/query.py

@@ -54,10 +54,6 @@ def _determine_backend(self):
 
         backend_alias = connection_router.for_read(**hints)
 
-        if isinstance(backend_alias, (list, tuple)) and len(backend_alias):
-            # We can only effectively read from one engine.
-            backend_alias = backend_alias[0]
-
         # The ``SearchQuery`` might swap itself out for a different variant
         # here.
         if self.query:

haystack/utils/loading.py

@@ -10,6 +10,7 @@
 
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
+from django.utils import six
 from django.utils.module_loading import module_has_submodule
 
 from haystack.exceptions import NotHandled, SearchFieldError
@@ -140,7 +141,7 @@ def routers(self):
                 self._routers.append(router_class())
         return self._routers
 
-    def for_action(self, action, **hints):
+    def _for_action(self, action, many, **hints):
         conns = []
 
         for router in self.routers:
@@ -149,15 +150,20 @@ def for_action(self, action, **hints):
                 connection_to_use = action_callable(**hints)
 
                 if connection_to_use is not None:
-                    conns.append(connection_to_use)
+                    if isinstance(connection_to_use, six.string_types):
+                        conns.append(connection_to_use)
+                    else:
+                        conns.extend(connection_to_use)
+                    if not many:
+                        break
 
         return conns
 
     def for_write(self, **hints):
-        return self.for_action('for_write', **hints)
+        return self._for_action('for_write', True, **hints)
 
     def for_read(self, **hints):
-        return self.for_action('for_read', **hints)
+        return self._for_action('for_read', False, **hints)[0]
 
 
 class UnifiedIndex(object):

test_haystack/mocks.py

@@ -32,6 +32,11 @@ def for_write(self, **hints):
         return None
 
 
+class MockMultiRouter(BaseRouter):
+    def for_write(self, **hints):
+        return ['multi1', 'multi2']
+
+
 class MockSearchResult(SearchResult):
     def __init__(self, app_label, model_name, pk, score, **kwargs):
         super(MockSearchResult, self).__init__(app_label, model_name, pk, score, **kwargs)

test_haystack/test_loading.py

@@ -111,25 +111,32 @@ def test_router_override3(self):
     def test_actions1(self):
         del settings.HAYSTACK_ROUTERS
         cr = loading.ConnectionRouter()
-        self.assertEqual(cr.for_read(), ['default'])
+        self.assertEqual(cr.for_read(), 'default')
         self.assertEqual(cr.for_write(), ['default'])
 
     @override_settings(HAYSTACK_ROUTERS=['test_haystack.mocks.MockMasterSlaveRouter', 'haystack.routers.DefaultRouter'])
     def test_actions2(self):
         cr = loading.ConnectionRouter()
-        self.assertEqual(cr.for_read(), ['slave', 'default'])
+        self.assertEqual(cr.for_read(), 'slave')
         self.assertEqual(cr.for_write(), ['master', 'default'])
 
     @override_settings(HAYSTACK_ROUTERS=['test_haystack.mocks.MockPassthroughRouter', 'test_haystack.mocks.MockMasterSlaveRouter', 'haystack.routers.DefaultRouter'])
     def test_actions3(self):
         cr = loading.ConnectionRouter()
         # Demonstrate pass-through
-        self.assertEqual(cr.for_read(), ['slave', 'default'])
+        self.assertEqual(cr.for_read(), 'slave')
         self.assertEqual(cr.for_write(), ['master', 'default'])
         # Demonstrate that hinting can change routing.
-        self.assertEqual(cr.for_read(pass_through=False), ['pass', 'slave', 'default'])
+        self.assertEqual(cr.for_read(pass_through=False), 'pass')
         self.assertEqual(cr.for_write(pass_through=False), ['pass', 'master', 'default'])
 
+    @override_settings(HAYSTACK_ROUTERS=['test_haystack.mocks.MockMultiRouter', 'haystack.routers.DefaultRouter'])
+    def test_actions4(self):
+        cr = loading.ConnectionRouter()
+        # Demonstrate that a router can return multiple backends in the "for_write" method
+        self.assertEqual(cr.for_read(), 'default')
+        self.assertEqual(cr.for_write(), ['multi1', 'multi2', 'default'])
+
 
 class MockNotAModel(object):
     pass