update with a more minimal fix and a better understanding of the

problem.

For the record:
  Our deadlock occurred when the __all_claimed check passed, but when
another thread released the last element in the pool before we took
back up the lock.  Doing the check inside of the lock (thus preventing
mutation of the claimed state for elements) closes the loophole for the
race.
  Making the pool lock re-entrant and setting the condition variable
might not be required, but it affords us a little bit of extra safety.

Author: evanmcc

riak/transports/pool.py

@@ -46,8 +46,6 @@ def __init__(self, obj):
         self.object = obj
         """Whether the resource is currently in use."""
         self.claimed = False
-        """Whether the object has been deleted."""
-        self.tomb = False
 
 
 class Pool(object):
@@ -84,8 +82,8 @@ def __init__(self):
         Creates a new Pool. This should be called manually if you
         override the __init__ method in a subclass.
         """
-        self.lock = threading.Lock()
-        self.releaser = threading.Condition()
+        self.lock = threading.RLock()
+        self.releaser = threading.Condition(self.lock)
         self.elements = list()
 
     @contextmanager
@@ -142,8 +140,8 @@ def delete_element(self, element):
         """
         with self.lock:
             self.elements.remove(element)
-            element.tomb = True
         self.destroy_resource(element.object)
+        del element
 
     def __iter__(self):
         """
@@ -205,20 +203,15 @@ def __iter__(self):
     def next(self):
         if len(self.targets) == 0:
             raise StopIteration
-        while len(self.unlocked) == 0:
+        if len(self.unlocked) == 0:
             self.__claim_elements()
-        ele = None
-        while not ele:
-            ele = self.unlocked.pop(0)
-            if ele.tomb:
-                ele = None
-        return ele
+        return self.unlocked.pop(0)
 
     def __claim_elements(self):
         with self.lock:
-            if self.__all_claimed():
-                with self.releaser:
-                    self.releaser.wait(.05)
+            with self.releaser:
+                if self.__all_claimed():
+                    self.releaser.wait()
             for element in self.targets:
                 if not element.claimed:
                     self.targets.remove(element)