CSHARP-1690: Fix race condition with BinaryConnection Dispose.

Author: rstam

src/MongoDB.Driver.Core/Core/Connections/BinaryConnection.cs

@@ -48,6 +48,7 @@ internal class BinaryConnection : IConnection
         private EndPoint _endPoint;
         private ConnectionDescription _description;
         private readonly Dropbox _dropbox = new Dropbox();
+        private bool _failedEventHasBeenRaised;
         private DateTime _lastUsedAtUtc;
         private DateTime _openedAtUtc;
         private readonly object _openLock = new object();
@@ -151,8 +152,18 @@ public ConnectionSettings Settings
         // methods
         private void ConnectionFailed(Exception exception)
         {
-            if (_state.TryChange(State.Open, State.Failed))
+            if (!_state.TryChange(State.Open, State.Failed))
             {
+                var currentState = _state.Value;
+                if (currentState != State.Failed && currentState != State.Disposed)
+                {
+                    throw new InvalidOperationException($"Invalid BinaryConnection state transition from {currentState} to Failed.");
+                }
+            }
+
+            if (!_failedEventHasBeenRaised)
+            {
+                _failedEventHasBeenRaised = true;
                 if (_failedEventHandler != null)
                 {
                     _failedEventHandler(new ConnectionFailedEvent(_connectionId, exception));
@@ -661,7 +672,14 @@ public OpenConnectionHelper(BinaryConnection connection)
 
             public void FailedOpeningConnection(Exception wrappedException)
             {
-                _connection._state.TryChange(State.Failed);
+                if (!_connection._state.TryChange(State.Connecting, State.Failed) && !_connection._state.TryChange(State.Initializing, State.Failed))
+                {
+                    var currentState = _connection._state.Value;
+                    if (currentState != State.Disposed)
+                    {
+                        throw new InvalidOperationException($"Invalid BinaryConnection state transition from {currentState} to Failed.");
+                    }
+                }
 
                 var handler = _connection._failedOpeningEventHandler;
                 if (handler != null)
@@ -672,14 +690,37 @@ public void FailedOpeningConnection(Exception wrappedException)
 
             public void InitializingConnection()
             {
-                _connection._state.TryChange(State.Initializing);
+                if (!_connection._state.TryChange(State.Connecting, State.Initializing))
+                {
+                    var currentState = _connection._state.Value;
+                    if (currentState == State.Disposed)
+                    {
+                        throw new ObjectDisposedException(typeof(BinaryConnection).Name);
+                    }
+                    else
+                    {
+                        throw new InvalidOperationException($"Invalid BinaryConnection state transition from {currentState} to Initializing.");
+                    }
+                }
             }
 
             public void OpenedConnection()
             {
                 _stopwatch.Stop();
                 _connection._connectionId = _connection._description.ConnectionId;
-                _connection._state.TryChange(State.Open);
+
+                if (!_connection._state.TryChange(State.Initializing, State.Open))
+                {
+                    var currentState = _connection._state.Value;
+                    if (currentState == State.Disposed)
+                    {
+                        throw new ObjectDisposedException(typeof(BinaryConnection).Name);
+                    }
+                    else
+                    {
+                        throw new InvalidOperationException($"Invalid BinaryConnection state transition from {currentState} to Open.");
+                    }
+                }
 
                 var handler = _connection._openedEventHandler;
                 if (handler != null)
@@ -883,6 +924,7 @@ public void SentMessages(int bufferLength)
 
         private static class State
         {
+            // note: the numeric values matter because sometimes we compare their magnitudes
             public static int Initial = 0;
             public static int Connecting = 1;
             public static int Initializing = 2;

src/MongoDB.Driver.Core/Core/Servers/ServerMonitor.cs

@@ -34,7 +34,7 @@ internal sealed class ServerMonitor : IServerMonitor
         private readonly ExponentiallyWeightedMovingAverage _averageRoundTripTimeCalculator = new ExponentiallyWeightedMovingAverage(0.2);
         private readonly ServerDescription _baseDescription;
         private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
-        private IConnection _connection;
+        private volatile IConnection _connection;
         private readonly IConnectionFactory _connectionFactory;
         private ServerDescription _currentDescription;
         private readonly EndPoint _endPoint;
@@ -136,25 +136,31 @@ private async Task<bool> HeartbeatAsync(CancellationToken cancellationToken)
             Exception heartbeatException = null;
             for (var attempt = 1; attempt <= maxRetryCount; attempt++)
             {
+                var connection = _connection;
                 try
                 {
-                    if (_connection == null)
+                    if (connection == null)
                     {
-                        _connection = _connectionFactory.CreateConnection(_serverId, _endPoint);
+                        connection = _connectionFactory.CreateConnection(_serverId, _endPoint);
                         // if we are cancelling, it's because the server has
                         // been shut down and we really don't need to wait.
-                        await _connection.OpenAsync(cancellationToken).ConfigureAwait(false);
+                        await connection.OpenAsync(cancellationToken).ConfigureAwait(false);
                     }
 
-                    heartbeatInfo = await GetHeartbeatInfoAsync(_connection, cancellationToken).ConfigureAwait(false);
+                    heartbeatInfo = await GetHeartbeatInfoAsync(connection, cancellationToken).ConfigureAwait(false);
                     heartbeatException = null;
+
+                    _connection = connection;
                     break;
                 }
                 catch (Exception ex)
                 {
                     heartbeatException = ex;
-                    _connection.Dispose();
                     _connection = null;
+                    if (connection != null)
+                    {
+                        connection.Dispose();
+                    }
                 }
             }