SERVER-50549 transform state-changing errors returned by mongos

Author: BillyDonahue

jstests/sharding/change_stream_error_label.js

@@ -42,6 +42,7 @@ const expectedStopShardErrors = [
 // First, verify that the 'failGetMoreAfterCursorCheckout' failpoint can effectively exercise the
 // error label generation logic for change stream getMores.
 function testFailGetMoreAfterCursorCheckoutFailpoint({errorCode, expectedLabel}) {
+    errorCode = ErrorCodes.doMongosRewrite(errorCode);
     // Activate the failpoint and set the exception that it will throw.
     assert.commandWorked(testDB.adminCommand({
         configureFailPoint: "failGetMoreAfterCursorCheckout",

jstests/sharding/linearizable_read_concern.js

@@ -85,7 +85,7 @@ var res = assert.commandFailed(testDB.runReadCommand({
     readConcern: {level: "linearizable"},
     maxTimeMS: shard0ReplTest.kDefaultTimeoutMS
 }));
-assert.eq(res.code, ErrorCodes.NotWritablePrimary);
+assert.eq(res.code, ErrorCodes.doMongosRewrite(ErrorCodes.NotWritablePrimary));
 
 jsTestLog("Testing linearizable read from primaries.");
 

jstests/sharding/mongos_not_retry_commands_in_transactions.js

@@ -54,15 +54,16 @@ jsTest.log(
     "Testing that mongos doesn't retry the read command with startTransaction=true on replication set failover.");
 assert.commandWorked(setCommandToFail(primaryConnection, "find", kNs));
 
-assert.commandFailedWithCode(mongosDB.runCommand({
-    find: kCollName,
-    filter: kDoc0,
-    startTransaction: true,
-    txnNumber: NumberLong(transactionNumber++),
-    stmtId: NumberInt(0),
-    autocommit: false
-}),
-                             ErrorCodes.InterruptedDueToReplStateChange);
+assert.commandFailedWithCode(
+    mongosDB.runCommand({
+        find: kCollName,
+        filter: kDoc0,
+        startTransaction: true,
+        txnNumber: NumberLong(transactionNumber++),
+        stmtId: NumberInt(0),
+        autocommit: false
+    }),
+    ErrorCodes.doMongosRewrite(ErrorCodes.InterruptedDueToReplStateChange));
 
 jsTest.log("Testing that mongos retries retryable writes on failover.");
 assert.commandWorked(setCommandToFail(primaryConnection, "insert", kNs));

jstests/sharding/retryable_write_error_labels.js

@@ -28,6 +28,7 @@ const st = new ShardingTest({
 });
 
 function checkErrorCode(res, errorCode, isWCError) {
+    errorCode = ErrorCodes.doMongosRewrite(errorCode);
     if (isWCError) {
         assert.neq(null, res.writeConcernError, res);
         assert.eq(res.writeConcernError.code, errorCode, res);

jstests/sharding/transactions_writes_not_retryable.js

@@ -31,7 +31,7 @@ function runTest(st, session, sessionDB, writeCmdName, writeCmd, isSharded) {
     session.startTransaction();
     assert.commandFailedWithCode(
         sessionDB.runCommand(writeCmd),
-        retryableError,
+        ErrorCodes.doMongosRewrite(retryableError),
         "expected write in transaction not to be retried on retryable error, cmd: " +
             tojson(writeCmd) + ", sharded: " + isSharded);
     assert.commandFailedWithCode(session.abortTransaction_forTesting(),

src/mongo/db/SConscript

@@ -601,6 +601,7 @@ env.Library(
         '$BUILD_DIR/mongo/db/query/explain_options',
         '$BUILD_DIR/mongo/idl/idl_parser',
         '$BUILD_DIR/mongo/rpc/command_status',
+        '$BUILD_DIR/mongo/rpc/rewrite_state_change_errors',
         '$BUILD_DIR/mongo/rpc/rpc',
         'audit',
         'auth/auth',

src/mongo/db/commands.cpp

@@ -61,6 +61,7 @@
 #include "mongo/rpc/metadata/client_metadata.h"
 #include "mongo/rpc/op_msg_rpc_impls.h"
 #include "mongo/rpc/protocol.h"
+#include "mongo/rpc/rewrite_state_change_errors.h"
 #include "mongo/rpc/write_concern_error_detail.h"
 #include "mongo/s/stale_exception.h"
 #include "mongo/util/fail_point.h"
@@ -681,6 +682,13 @@ void CommandHelpers::evaluateFailCommandFailPoint(OperationContext* opCtx,
     const Command* cmd = invocation->definition();
     failCommand.executeIf(
         [&](const BSONObj& data) {
+            // State change codes are rewritten on the way out of a `mongos`
+            // server. Errors injected via failpoint manipulation are normally
+            // exempt from this. However, we provide an override option so they
+            // can be made subject to rewriting if that's really necessary.
+            if (bool b; !bsonExtractBooleanField(data, "allowRewriteStateChange", &b).isOK() || !b)
+                rpc::RewriteStateChangeErrors::setEnabled(opCtx, false);
+
             if (data.hasField(kErrorLabelsFieldName) &&
                 data[kErrorLabelsFieldName].type() == Array) {
                 // Propagate error labels specified in the failCommand failpoint to the

src/mongo/db/commands/SConscript

@@ -360,6 +360,7 @@ env.Library(
         '$BUILD_DIR/mongo/db/views/views_mongod',
         '$BUILD_DIR/mongo/executor/async_request_executor',
         '$BUILD_DIR/mongo/idl/feature_flag',
+        '$BUILD_DIR/mongo/rpc/rewrite_state_change_errors',
         '$BUILD_DIR/mongo/util/log_and_backoff',
         '$BUILD_DIR/mongo/util/net/http_client',
         'core',

src/mongo/db/commands/getmore_cmd.cpp

@@ -36,6 +36,7 @@
 #include <memory>
 #include <string>
 
+#include "mongo/bson/util/bson_extract.h"
 #include "mongo/db/api_parameters.h"
 #include "mongo/db/auth/authorization_checks.h"
 #include "mongo/db/auth/authorization_session.h"
@@ -64,6 +65,7 @@
 #include "mongo/db/stats/resource_consumption_metrics.h"
 #include "mongo/db/stats/top.h"
 #include "mongo/logv2/log.h"
+#include "mongo/rpc/rewrite_state_change_errors.h"
 #include "mongo/s/chunk_version.h"
 #include "mongo/util/fail_point.h"
 #include "mongo/util/scopeguard.h"
@@ -575,7 +577,11 @@ class GetMoreCmd final : public Command {
             // If the 'failGetMoreAfterCursorCheckout' failpoint is enabled, throw an exception with
             // the given 'errorCode' value, or ErrorCodes::InternalError if 'errorCode' is omitted.
             failGetMoreAfterCursorCheckout.executeIf(
-                [](const BSONObj& data) {
+                [&](const BSONObj& data) {
+                    if (bool b;
+                        !bsonExtractBooleanField(data, "allowRewriteStateChange", &b).isOK() || !b)
+                        rpc::RewriteStateChangeErrors::setEnabled(opCtx, false);
+
                     auto errorCode = (data["errorCode"] ? data["errorCode"].safeNumberLong()
                                                         : ErrorCodes::InternalError);
                     uasserted(errorCode, "Hit the 'failGetMoreAfterCursorCheckout' failpoint");

src/mongo/rpc/SConscript

@@ -74,6 +74,23 @@ env.Library(
     ],
     LIBDEPS_PRIVATE=[
         '$BUILD_DIR/mongo/idl/server_parameter',
+        'rewrite_state_change_errors',
+    ],
+)
+
+env.Library(
+    target='rewrite_state_change_errors',
+    source=[
+        'rewrite_state_change_errors.cpp',
+        'rewrite_state_change_errors_server_parameter.idl',
+    ],
+    LIBDEPS_PRIVATE=[
+        '$BUILD_DIR/mongo/base',
+        '$BUILD_DIR/mongo/bson/mutable/mutable_bson',
+        '$BUILD_DIR/mongo/db/service_context',
+        '$BUILD_DIR/mongo/rpc/protocol',
+        '$BUILD_DIR/mongo/s/is_mongos',
+        '$BUILD_DIR/third_party/shim_pcrecpp',
     ],
 )
 
@@ -170,12 +187,14 @@ if wiredtiger:
             'op_msg_test.cpp',
             'protocol_test.cpp',
             'reply_builder_test.cpp',
+            'rewrite_state_change_errors_test.cpp',
         ],
         LIBDEPS=[
             '$BUILD_DIR/mongo/client/clientdriver_minimal',
             '$BUILD_DIR/third_party/wiredtiger/wiredtiger_checksum',
             'client_metadata',
             'metadata',
+            'rewrite_state_change_errors',
             'rpc',
         ]
     )