Skip to content

Commit 40e8c73

Browse files
BorisDogBorisDog
committed
CSHARP-3442: Provide explicit guidance on handling command errors that occur before the handshake completes during operation execution
1 parent 9611327 commit 40e8c73

34 files changed

+2482
-26
lines changed

‎src/MongoDB.Driver.Core/Core/Operations/RetryabilityHelper.cs

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,8 @@ static RetryabilityHelper()
4040
var resumableAndRetryableExceptions = new HashSet<Type>()
4141
{
4242
typeof(MongoNotPrimaryException),
43-
typeof(MongoNodeIsRecoveringException)
43+
typeof(MongoNodeIsRecoveringException),
44+
typeof(MongoConnectionPoolPausedException)
4445
};
4546

4647
__resumableChangeStreamExceptions = new HashSet<Type>(resumableAndRetryableExceptions);
@@ -96,6 +97,14 @@ public static void AddRetryableWriteErrorLabelIfRequired(MongoException exceptio
9697
}
9798
}
9899

100+
public static void AddRetryableWriteErrorLabelOnCheckoutRetryableWrite(Exception exception)
101+
{
102+
if (exception is MongoConnectionPoolPausedException mongoPoolPausedException)
103+
{
104+
mongoPoolPausedException.AddErrorLabel(RetryableWriteErrorLabel);
105+
}
106+
}
107+
99108
public static bool IsCommandRetryable(BsonDocument command)
100109
{
101110
return

‎src/MongoDB.Driver.Core/Core/Operations/RetryableReadContext.cs

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -177,13 +177,47 @@ public void ReplaceChannelSource(IChannelSourceHandle channelSource)
177177
private void Initialize(CancellationToken cancellationToken)
178178
{
179179
_channelSource = _binding.GetReadChannelSource(cancellationToken);
180-
_channel = _channelSource.GetChannel(cancellationToken);
180+
var serverDescription = _channelSource.ServerDescription;
181+
182+
try
183+
{
184+
_channel = _channelSource.GetChannel(cancellationToken);
185+
}
186+
catch (MongoConnectionPoolPausedException)
187+
{
188+
if (RetryableReadOperationExecutor.ShouldConnectionAcquireBeRetried(this, serverDescription))
189+
{
190+
ReplaceChannelSource(_binding.GetReadChannelSource(cancellationToken));
191+
ReplaceChannel(_channelSource.GetChannel(cancellationToken));
192+
}
193+
else
194+
{
195+
throw;
196+
}
197+
}
181198
}
182199

183200
private async Task InitializeAsync(CancellationToken cancellationToken)
184201
{
185202
_channelSource = await _binding.GetReadChannelSourceAsync(cancellationToken).ConfigureAwait(false);
186-
_channel = await _channelSource.GetChannelAsync(cancellationToken).ConfigureAwait(false);
203+
var serverDescription = _channelSource.ServerDescription;
204+
205+
try
206+
{
207+
_channel = await _channelSource.GetChannelAsync(cancellationToken).ConfigureAwait(false);
208+
}
209+
catch (MongoConnectionPoolPausedException)
210+
{
211+
if (RetryableReadOperationExecutor.ShouldConnectionAcquireBeRetried(this, serverDescription))
212+
{
213+
ReplaceChannelSource(await _binding.GetReadChannelSourceAsync(cancellationToken).ConfigureAwait(false));
214+
ReplaceChannel(await _channelSource.GetChannelAsync(cancellationToken).ConfigureAwait(false));
215+
}
216+
else
217+
{
218+
throw;
219+
}
220+
}
187221
}
188222
}
189223
}

‎src/MongoDB.Driver.Core/Core/Operations/RetryableReadOperationExecutor.cs

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@
1717
using System.Threading;
1818
using System.Threading.Tasks;
1919
using MongoDB.Driver.Core.Bindings;
20-
using MongoDB.Driver.Core.Connections;
2120
using MongoDB.Driver.Core.Misc;
21+
using MongoDB.Driver.Core.Servers;
2222

2323
namespace MongoDB.Driver.Core.Operations
2424
{
@@ -138,6 +138,13 @@ public static async Task<TResult> ExecuteAsync<TResult>(IRetryableReadOperation<
138138
}
139139
}
140140

141+
public static bool ShouldConnectionAcquireBeRetried(RetryableReadContext context, ServerDescription serverDescription)
142+
{
143+
return context.RetryRequested &&
144+
Feature.RetryableReads.IsSupported(serverDescription.Version) &&
145+
!context.Binding.Session.IsInTransaction;
146+
}
147+
141148
// private static methods
142149
private static bool AreRetryableReadsSupported(RetryableReadContext context)
143150
{

‎src/MongoDB.Driver.Core/Core/Operations/RetryableWriteContext.cs

Lines changed: 36 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -195,13 +195,47 @@ public void ReplaceChannelSource(IChannelSourceHandle channelSource)
195195
private void Initialize(CancellationToken cancellationToken)
196196
{
197197
_channelSource = _binding.GetWriteChannelSource(cancellationToken);
198-
_channel = _channelSource.GetChannel(cancellationToken);
198+
var serverDescription = _channelSource.ServerDescription;
199+
200+
try
201+
{
202+
_channel = _channelSource.GetChannel(cancellationToken);
203+
}
204+
catch (MongoConnectionPoolPausedException)
205+
{
206+
if (RetryableWriteOperationExecutor.ShouldConnectionAcquireBeRetried(this, serverDescription))
207+
{
208+
ReplaceChannelSource(_binding.GetWriteChannelSource(cancellationToken));
209+
ReplaceChannel(_channelSource.GetChannel(cancellationToken));
210+
}
211+
else
212+
{
213+
throw;
214+
}
215+
}
199216
}
200217

201218
private async Task InitializeAsync(CancellationToken cancellationToken)
202219
{
203220
_channelSource = await _binding.GetWriteChannelSourceAsync(cancellationToken).ConfigureAwait(false);
204-
_channel = await _channelSource.GetChannelAsync(cancellationToken).ConfigureAwait(false);
221+
var serverDescription = _channelSource.ServerDescription;
222+
223+
try
224+
{
225+
_channel = await _channelSource.GetChannelAsync(cancellationToken).ConfigureAwait(false);
226+
}
227+
catch (MongoConnectionPoolPausedException)
228+
{
229+
if (RetryableWriteOperationExecutor.ShouldConnectionAcquireBeRetried(this, serverDescription))
230+
{
231+
ReplaceChannelSource(await _binding.GetWriteChannelSourceAsync(cancellationToken).ConfigureAwait(false));
232+
ReplaceChannel(await _channelSource.GetChannelAsync(cancellationToken).ConfigureAwait(false));
233+
}
234+
else
235+
{
236+
throw;
237+
}
238+
}
205239
}
206240
}
207241
}

‎src/MongoDB.Driver.Core/Core/Operations/RetryableWriteOperationExecutor.cs

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,15 @@ public static async Task<TResult> ExecuteAsync<TResult>(IRetryableWriteOperation
127127
}
128128
}
129129

130-
// privates static methods
130+
public static bool ShouldConnectionAcquireBeRetried(RetryableWriteContext context, ServerDescription serverDescription)
131+
{
132+
return context.RetryRequested &&
133+
AreRetryableWritesSupported(serverDescription) &&
134+
context.Binding.Session.Id != null &&
135+
!context.Binding.Session.IsInTransaction;
136+
}
137+
138+
// private static methods
131139
private static bool AreRetriesAllowed<TResult>(IRetryableWriteOperation<TResult> operation, RetryableWriteContext context)
132140
{
133141
return IsOperationAcknowledged(operation) && DoesContextAllowRetries(context);
@@ -141,6 +149,12 @@ private static bool AreRetryableWritesSupported(ConnectionDescription connection
141149
(helloResult.LogicalSessionTimeout != null && helloResult.ServerType != ServerType.Standalone);
142150
}
143151

152+
private static bool AreRetryableWritesSupported(ServerDescription serverDescription)
153+
{
154+
return serverDescription.Type == ServerType.LoadBalanced ||
155+
(serverDescription.LogicalSessionTimeout != null && serverDescription.Type != ServerType.Standalone);
156+
}
157+
144158
private static bool DoesContextAllowRetries(RetryableWriteContext context)
145159
{
146160
return
@@ -158,9 +172,8 @@ private static bool IsOperationAcknowledged<TResult>(IRetryableWriteOperation<TR
158172
writeConcern.IsAcknowledged;
159173
}
160174

161-
private static bool ShouldThrowOriginalException(Exception retryException)
162-
{
163-
return retryException is MongoException && !(retryException is MongoConnectionException);
164-
}
175+
private static bool ShouldThrowOriginalException(Exception retryException) =>
176+
retryException == null ||
177+
retryException is MongoException && !(retryException is MongoConnectionException || retryException is MongoConnectionPoolPausedException);
165178
}
166179
}

‎src/MongoDB.Driver.Core/Core/Operations/UpdateOpcodeOperation.cs

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,9 @@
1717
using System.Threading;
1818
using System.Threading.Tasks;
1919
using MongoDB.Driver.Core.Bindings;
20-
using MongoDB.Driver.Core.Connections;
2120
using MongoDB.Driver.Core.Events;
2221
using MongoDB.Driver.Core.Misc;
2322
using MongoDB.Driver.Core.Operations.ElementNameValidators;
24-
using MongoDB.Driver.Core.WireProtocol;
2523
using MongoDB.Driver.Core.WireProtocol.Messages.Encoders;
2624

2725
namespace MongoDB.Driver.Core.Operations

‎tests/MongoDB.Driver.Core.TestHelpers/EventCapturer.cs

Lines changed: 26 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
using System;
1717
using System.Collections.Generic;
1818
using System.Linq;
19-
using System.Threading;
2019
using System.Threading.Tasks;
2120
using MongoDB.Bson;
2221
using MongoDB.Bson.IO;
@@ -62,6 +61,17 @@ public EventCapturer Capture<TEvent>(Func<TEvent, bool> predicate = null)
6261
return this;
6362
}
6463

64+
public EventCapturer CaptureCommandEvents(string commandName = null)
65+
{
66+
Func<string, bool> predicate = s => commandName == null || commandName == s;
67+
68+
_eventsToCapture.Add(typeof(CommandStartedEvent), o => predicate(((CommandStartedEvent)o).CommandName));
69+
_eventsToCapture.Add(typeof(CommandSucceededEvent), o => predicate(((CommandSucceededEvent)o).CommandName));
70+
_eventsToCapture.Add(typeof(CommandFailedEvent), o => predicate(((CommandFailedEvent)o).CommandName));
71+
72+
return this;
73+
}
74+
6575
public int Count
6676
{
6777
get
@@ -142,11 +152,23 @@ public void WaitForOrThrowIfTimeout(Func<object, bool>[] matchSequence, TimeSpan
142152
{
143153
Func<IEnumerable<object>, bool> condition = @events =>
144154
{
145-
var enumerator = @events.GetEnumerator();
155+
var allEvents = @events.ToArray();
156+
var maxEventIndex = allEvents.Length - matchSequence.Length;
146157

147-
while (enumerator.MoveNext())
158+
for (int i = 0; i <= maxEventIndex; i++)
148159
{
149-
if (matchSequence.All(m => m(enumerator.Current) && enumerator.MoveNext()))
160+
var allMatched = true;
161+
162+
for (int j = 0; j < matchSequence.Length; j++)
163+
{
164+
if (!matchSequence[j](allEvents[i + j]))
165+
{
166+
allMatched = false;
167+
break;
168+
}
169+
}
170+
171+
if (allMatched)
150172
{
151173
return true;
152174
}

‎tests/MongoDB.Driver.Core.Tests/Specifications/connection-monitoring-and-pooling/tests/pool-create-min-size-error.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ failPoint:
1010
# high amount to ensure not interfered with by monitor checks.
1111
mode: { times: 50 }
1212
data:
13-
failCommands: ["isMaster"]
13+
failCommands: ["isMaster","hello"]
1414
closeConnection: true
1515
appName: "poolCreateMinSizeErrorTest"
1616
poolOptions:

‎tests/MongoDB.Driver.Core.Tests/Specifications/connection-monitoring-and-pooling/tests/wait-queue-timeout.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"description": "must aggressively timeout threads enqueued longer than waitQueueTimeoutMS",
55
"poolOptions": {
66
"maxPoolSize": 1,
7-
"waitQueueTimeoutMS": 20
7+
"waitQueueTimeoutMS": 50
88
},
99
"operations": [
1010
{

‎tests/MongoDB.Driver.Core.Tests/Specifications/connection-monitoring-and-pooling/tests/wait-queue-timeout.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ style: unit
33
description: must aggressively timeout threads enqueued longer than waitQueueTimeoutMS
44
poolOptions:
55
maxPoolSize: 1
6-
waitQueueTimeoutMS: 20
6+
waitQueueTimeoutMS: 50
77
operations:
88
- name: ready
99
# Check out only possible connection

‎tests/MongoDB.Driver.Tests/Specifications/Runner/MongoClientJsonDrivenTestRunnerBase.cs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,8 @@ protected virtual bool TryConfigureClientOption(MongoClientSettings settings, Bs
332332
break;
333333

334334
case "connectTimeoutMS":
335-
settings.ConnectTimeout = TimeSpan.FromMilliseconds(option.Value.ToInt32());
335+
var connectTimeoutMS = option.Value.ToInt32();
336+
settings.ConnectTimeout = connectTimeoutMS == 0 ? Timeout.InfiniteTimeSpan : TimeSpan.FromMilliseconds(connectTimeoutMS);
336337
break;
337338

338339
case "directConnection":

‎tests/MongoDB.Driver.Tests/Specifications/load-balancers/tests/wait-queue-timeouts.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
"useMultipleMongoses": true,
1616
"uriOptions": {
1717
"maxPoolSize": 1,
18-
"waitQueueTimeoutMS": 5
18+
"waitQueueTimeoutMS": 50
1919
},
2020
"observeEvents": [
2121
"connectionCheckedOutEvent",

‎tests/MongoDB.Driver.Tests/Specifications/load-balancers/tests/wait-queue-timeouts.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ createEntities:
1111
useMultipleMongoses: true
1212
uriOptions:
1313
maxPoolSize: 1
14-
waitQueueTimeoutMS: 5
14+
waitQueueTimeoutMS: 50
1515
observeEvents:
1616
- connectionCheckedOutEvent
1717
- connectionCheckOutFailedEvent

0 commit comments

Comments
 (0)