SERVER-20580 Re-enable csrs_upgrade_during_migrate.js test

Author: stbrody

buildscripts/resmokeconfig/suites/sharding.yml

@@ -4,7 +4,6 @@ selector:
     - jstests/sharding/*.js
     - jstests/sharding/replset_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade_during_migrate.js     # flaky - SERVER-20580
 
 executor:
   js_test:

buildscripts/resmokeconfig/suites/sharding_auth.yml

@@ -10,7 +10,6 @@ selector:
     - jstests/sharding/replset_config/*.js
     - jstests/sharding/sync_cluster_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade_during_migrate.js    # flaky - SERVER-20580
     # Skip any tests that run with auth explicitly.
     - jstests/sharding/*[aA]uth*.js
     - jstests/sharding/replset_config/*[aA]uth*.js

buildscripts/resmokeconfig/suites/sharding_auth_audit.yml

@@ -10,7 +10,6 @@ selector:
     - jstests/sharding/replset_config/*.js
     - jstests/sharding/sync_cluster_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade_during_migrate.js    # flaky - SERVER-20580
     # Skip any tests that run with auth explicitly.
     - jstests/sharding/*[aA]uth*.js
     - jstests/sharding/replset_config/*[aA]uth*.js

buildscripts/resmokeconfig/suites/sharding_continuous_config_stepdown.yml

@@ -4,10 +4,6 @@ selector:
     - jstests/sharding/*.js
     - jstests/sharding/replset_config/*.js
     exclude_files:
-    # SERVER-20694
-    - jstests/sharding/csrs_upgrade.js
-    # Test is flaky - SERVER-20580
-    - jstests/sharding/csrs_upgrade_during_migrate.js
     # Auth tests require authentication on the stepdown thread's connection
     - jstests/sharding/auth*.js
     - jstests/sharding/cleanup_orphaned_auth.js

buildscripts/resmokeconfig/suites/sharding_ese.yml

@@ -8,7 +8,6 @@ selector:
     - jstests/sharding/*.js
     - jstests/sharding/replset_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade_during_migrate.js      # flaky - SERVER-20580
 
 executor:
   js_test:

buildscripts/resmokeconfig/suites/sharding_legacy.yml

@@ -4,7 +4,6 @@ selector:
     - jstests/sharding/*.js
     - jstests/sharding/sync_cluster_config/*.js
     exclude_files:
-    - jstests/sharding/csrs_upgrade_during_migrate.js  # SERVER-20580
 
 executor:
   js_test:

jstests/replsets/rslib.js

@@ -1,4 +1,6 @@
 var wait, occasionally, reconnect, getLatestOp, waitForAllMembers, reconfig, awaitOpTime;
+var waitUntilAllNodesCaughtUp;
+
 (function () {
 "use strict";
 var count = 0;
@@ -148,4 +150,35 @@ awaitOpTime = function (node, opTime) {
     });
 };
 
+/**
+ * Uses the results of running replSetGetStatus against an arbitrary replset node to wait until
+ * all nodes in the set are replicated through the same optime.
+ */
+waitUntilAllNodesCaughtUp = function(rs) {
+    var rsStatus;
+    var firstConflictingIndex;
+    var ot;
+    var otherOt;
+    assert.soon(function () {
+        rsStatus = rs[0].adminCommand('replSetGetStatus');
+        if (rsStatus.ok != 1) {
+            return false;
+        }
+        assert.eq(rs.length, rsStatus.members.length, tojson(rsStatus));
+        ot = rsStatus.members[0].optime;
+        for (var i = 1; i < rsStatus.members.length; ++i) {
+            otherOt = rsStatus.members[i].optime;
+            if (bsonWoCompare({ts: otherOt.ts}, {ts: ot.ts}) ||
+                bsonWoCompare({t: otherOt.t},  {t: ot.t})) {
+                firstConflictingIndex = i;
+                return false;
+            }
+        }
+        return true;
+    }, function () {
+        return "Optimes of members 0 (" + tojson(ot) + ") and " + firstConflictingIndex + " (" +
+            tojson(otherOt) + ") are different in " + tojson(rsStatus);
+    });
+};
+
 }());

jstests/sharding/csrs_upgrade.js

@@ -14,6 +14,9 @@
  * config.version is read to confirm the availability of metadata
  * reads.
  */
+
+load("jstests/replsets/rslib.js");
+
 var st;
 (function() {
     "use strict";
@@ -96,33 +99,6 @@ var st;
         }
     });
 
-    var waitUntilAllCaughtUp = function(csrs) {
-        var rsStatus;
-        var firstConflictingIndex;
-        var ot;
-        var otherOt;
-        assert.soon(function () {
-            rsStatus = csrs[0].adminCommand('replSetGetStatus');
-            if (rsStatus.ok != 1) {
-                return false;
-            }
-            assert.eq(csrs.length, rsStatus.members.length, tojson(rsStatus));
-            ot = rsStatus.members[0].optime;
-            for (var i = 1; i < rsStatus.members.length; ++i) {
-                otherOt = rsStatus.members[i].optime;
-                if (bsonWoCompare({ts: otherOt.ts}, {ts: ot.ts}) ||
-                    bsonWoCompare({t: otherOt.t},  {t: ot.t})) {
-                    firstConflictingIndex = i;
-                    return false;
-                }
-            }
-            return true;
-        }, function () {
-            return "Optimes of members 0 (" + tojson(ot) + ") and " + firstConflictingIndex + " (" +
-                tojson(otherOt) + ") are different in " + tojson(rsStatus);
-        });
-    };
-
     var shardConfigs = st.s0.getCollection("config.shards").find().toArray();
     assert.eq(2, shardConfigs.length);
     var shard0Name = shardConfigs[0]._id;
@@ -192,7 +168,7 @@ var st;
     assertOpsWork(st.s0, "using SCCC protocol when first config server is primary of " +
                   csrs.length + "-node replica set");
 
-    waitUntilAllCaughtUp(csrs);
+    waitUntilAllNodesCaughtUp(csrs);
 
     jsTest.log("Shutting down second and third SCCC config server nodes");
     MongoRunner.stopMongod(st.c1);

jstests/sharding/csrs_upgrade_during_migrate.js

@@ -120,10 +120,7 @@ var st;
     jsTest.log("Adding non-voting members to csrs set: " + tojson(csrsConfig));
     assert.commandWorked(csrs[0].adminCommand({replSetReconfig: csrsConfig}));
 
-    // This write is an easy way to wait for all members of the CSRS set to have
-    // replicated all of the documents.
-    assert.writeOK(csrs[0].getCollection('config.tmp').insert({},
-                                                              { writeConcern: { w:csrs.length }}));
+    waitUntilAllNodesCaughtUp(csrs);
 
     jsTest.log("Starting long-running chunk migration");
     // Turn on fail point to confirm that moveChunk aborts before getting to the critical section.
@@ -145,8 +142,9 @@ var st;
                     return st.s0.getDB('config').changelog.findOne({what: 'moveChunk.start'});
                 });
 
-    jsTest.log("Shutting down second and third SCCC config server nodes");
-    MongoRunner.stopMongod(st.c1);
+    // Only shut down one of the SCCC config servers to avoid any period without any config servers
+    // online.
+    jsTest.log("Shutting down third SCCC config server node");
     MongoRunner.stopMongod(st.c2);
 
     csrsConfig.members.forEach(function (member) { member.votes = 1; member.priority = 1});