Improve ndb_rpl_circular_2ch_rep_status reliability.

Testcase synchronisation issue - needed to wait for slave-applied 
changes to be binlogged before replication cutover.

Existing get_ndb_epochs.inc and wait_for_ndb_committed_to_binlog.inc 
backported and hacked to work in 7.1 to achieve this.

Author: frazerclement

mysql-test/include/get_ndb_epochs.inc

@@ -0,0 +1,63 @@
+# ==== Purpose ====
+#
+# Extract the epoch values from SHOW ENGINE NDB STATUS.
+#
+# ==== Usage ====
+#
+# --source include/get_ndb_epochs.inc
+# --echo $ndb_latest_epoch
+# --echo $ndb_latest_trans_epoch
+# --echo $ndb_latest_received_binlog_epoch
+# --echo $ndb_latest_handled_binlog_epoch
+# --echo $ndb_latest_applied_binlog_epoch
+#
+# ==== Parameters ====
+#
+# $ndb_status_binlog_row
+#   The row number of the output of SHOW ENGINE NDB STATUS
+#   last was seen.
+#   No need to set explictly, but used between sourcing this
+#   include file several times, since it is highly probable
+#   (but not certain) that same row number also applies next
+#   time.
+
+if (!$ndb_status_binlog_row)
+{
+  let $ndb_status_binlog_row = 1;
+}
+
+# Try two times, in case the row number of binlog state changes
+# between two invocations of SHOW ENGINE NDB STATUS.
+let $attempts= 20;
+let $found= 0;
+while ($attempts)
+{
+  let $q= query_get_value(SHOW ENGINE NDB STATUS, Status, $ndb_status_binlog_row);
+  if (`SELECT INSTR('$q', 'latest_epoch=')`)
+  {
+    let $attempts = 1;
+    let $found = 1;
+  }
+
+  dec $attempts;
+
+  if (!$attempts)
+  {
+    if (!$found)
+    {
+      echo ERROR IN TEST: No binlog row found in SHOW ENGINE NDB STATUS;
+      die ERROR IN TEST: No binlog row found in SHOW ENGINE NDB STATUS;
+    }
+  }
+
+  if (!$found)
+  {
+    inc $ndb_status_binlog_row;
+  }
+}
+
+let $ndb_latest_epoch= `SELECT SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING('$q', INSTR('$q', 'latest_epoch=')), ',', 1), '=', -1)`;
+let $ndb_latest_trans_epoch= `SELECT SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING('$q', INSTR('$q', 'latest_trans_epoch=')), ',', 1), '=', -1)`;
+let $ndb_latest_received_binlog_epoch= `SELECT SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING('$q', INSTR('$q', 'latest_received_binlog_epoch=')), ',', 1), '=', -1)`;
+let $ndb_latest_handled_binlog_epoch= `SELECT SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING('$q', INSTR('$q', 'latest_handled_binlog_epoch=')), ',', 1), '=', -1)`;
+let $ndb_latest_applied_binlog_epoch= `SELECT SUBSTRING_INDEX(SUBSTRING_INDEX(SUBSTRING('$q', INSTR('$q', 'latest_applied_binlog_epoch=')), ',', 1), '=', -1)`;

mysql-test/include/wait_for_ndb_committed_to_binlog.inc

@@ -0,0 +1,57 @@
+# ==== Purpose ====
+# 
+# Several test primitives from mysql-test/extra/rpl_tests 
+# are shared for test cases for MyISAM, InnoDB, NDB and
+# other engines.
+# For NDB engine all events will be added by NDB injector
+# so tests only can continue after injector is ready, 
+# this file waits for all committed transactions (at start
+# of file) are injected into the binlog.
+#
+# ==== Usage ====
+#
+# let $wait_timeout= 150;
+# --source include/wait_for_ndb_committed_to_binlog.inc
+#
+# ==== Parameters =====
+#
+# $wait_timeout
+#   Time to wait in seconds for epoch to be injected in binlog.
+#   On timeout test dies.
+#
+
+if (`SELECT 1 FROM information_schema.engines WHERE engine = 'ndbcluster' AND support IN ('YES', 'DEFAULT')`)
+{
+  let $max_run_time= 150;
+  if ($wait_timeout)
+  {
+    let $max_run_time= $wait_timeout;
+  }
+  # Reset $wait_timeout so that its value won't be used on subsequent
+  # calls, and default will be used instead.
+  let $wait_timeout= 0;
+
+  # The smallest timespan till UNIX_TIMESTAMP() gets incremented is ~0 seconds.
+  # We add one second to avoid the case that somebody measures timespans on a
+  # real clock with fractions of seconds, detects that n seconds are sufficient,
+  # assigns n to this routine and suffers because he sometimes gets n - 1
+  # seconds in reality.
+
+  let $max_end_time= `SELECT UNIX_TIMESTAMP() + $max_run_time + 1`;
+
+  source include/get_ndb_epochs.inc;
+
+  while (`SELECT $ndb_latest_trans_epoch > $ndb_latest_handled_binlog_epoch AND UNIX_TIMESTAMP() <= $max_end_time`)
+  {
+    # Sleep a bit to avoid too heavy load.
+    sleep 0.1;
+
+    source include/get_ndb_epochs.inc;
+  }
+
+  if (`SELECT $ndb_latest_trans_epoch > $ndb_latest_handled_binlog_epoch`)
+  {
+    echo ERROR IN TEST: Timeout in waiting for epoch $ndb_latest_trans_epoch to be handled, last handled is $ndb_latest_handled_binlog_epoch;
+    die ERROR IN TEST: Timeout in waiting for epoch to be handled;
+  }
+}

mysql-test/suite/ndb_rpl/t/ndb_rpl_circular_2ch_rep_status.test

@@ -70,6 +70,15 @@ select
 
 --echo Now wait for all replication to quiesce
 
+# Ensure that already slave-applied events from Cluster B 
+# are in-binlog before allowing the switchover. 
+# Otherwise we can setup replication from a binlog
+# pos before the already-applied events.
+# Symptom is that we get an extra ndb_apply_status line
+# for server_id 3.
+#
+--source include/wait_for_ndb_committed_to_binlog.inc
+
 --echo Now swap replication channels around
 --source include/rpl_stop_slaves.inc
 --let $rpl_topology= 2->1,3->4