WL#13574 Include MDL and ACL locks in MTS deadlock detection infra-structure [post-push]

Description
-----------
Three issues introduced by WL#13574 patch being addressed in this patch:

1) Multi-threaded applier stopping with a segmentation fault in ARM
   environments.

2) Pure virtual method invocation error in MDL graph infra-structure, being
   sporadically observed.

3) The `rpl_gtid.rpl_gtid_mts_spco_deadlock_other_locks` test-case is
   failing inn 8.0 and trunk branches.

Analysis/Fix
------------
The analysis and proposed fix for each of the above issues:

1) Within the `memory::Aligned_atomic` the L1 cache-line size is being
   fetched programtically and at runtime, in order to optimize memory
   usage. The method by which this configuratio value is acquired differs
   from OS to OS. For Linux, the method being used was to read a file from
   the `proc` filesystem. This is not portable, for instance in ARM such
   file doesn't exist. The proper way is to use `sysconf()` and the tag
   `_SC_LEVEL1_DCACHE_LINESIZE`.

2) In `Commit_order_manager::wait_on_graph` method, a
   `Commit_order_lock_graph` local object is being created, `ticket`, which
   reference is passed on to the MDL graph as a node to wait for. In the
   same method, a `raii::Sentry<>` object is created, in order to clean the
   `ticket` variable reference from the MDL graph at the end of the
   scope. The `Commit_order_lock_graph` reference stored in the MDL graph
   is accessed by every thread that executes a deadlock search on the MDL
   graph.
   The problem was that the `raii::Sentry<>` object was being instantiated
   **before** the `Commit_order_lock_graph` object. Since the order of
   disposal is inverse to the order of creation, the
   `Commit_order_lock_graph` object was being disposed of prior to the
   invocation of the clean up by the `raii::Senty<>`. So, a time hiatus
   between both disposals where the object was already disposed of but
   still referenced by the MDL graph. This is fixed by inverting the order
   of creation between the objects.
   Why not a segmentation fault?  Because, since the
   `Commit_order_lock_graph` object is local, the memory is still there, in
   the stack. OTOH, the destructor for the object was already invoked,
   meaning, no memory violation but the information about the object is
   cleared, hence the _pure virtual method invocation_ error due object
   access by casting it (the `Commit_order_lock_graph` reference) to the
   parent class, `MDL_wait_for_subgraph`.

3) Within a multi-threaded applier with _replica-preserve-commit-order_
   enabled, there are two execution paths by which the multi-threaded
   applier coordinator may exit due to a deadlock: all workers are waiting
   on the commit order queue and they are all asked to back-off by the MDL
   graph infra-structure; there are workers that haven't arrive the commit
   stage yet and will back-off due to the state of the commit order
   queue. The different paths make the applier output different error
   messages. The test-cases needed to be updated to reflect the
   difference. For each test case, both execution paths are now exercised
   and tested.

Reviewed-by: Pedro Gomes <[email protected]>
Reviewed-by: Sven Sandberg <[email protected]>
RB: 25616

Author: Pedro Figueiredo

mysql-test/extra/rpl_tests/mts_spco_generate_deadlock.inc

@@ -22,6 +22,20 @@
 #  The statement to be execute by one of the workers that should block the
 #  client connection.
 #
+# [--let $mts_spco_gd_worker_3_only_runs_after_deadlock = 0|1]
+#  Whether or not worker 3 should only proceed with the transaction
+#  execution after the deadlocak has been detected. If `0`, worker 3 will
+#  wait on the commit order queue before the deadlock is detected, leading
+#  to the use-case where both workers 2 and 3 will be force to rollback by
+#  the MDL graph infra-structure. If `1`, worker 3 will only start the
+#  transaction execution after the deadlock is detected and the coordinator
+#  already started the shutdown process, leading to the use-case where
+#  worker 2 is forced to rollback by the MDL graph infra-structure but
+#  worker 3 will rollback due to the commit queue infra-structure. These
+#  two use-cases generate a different set of error messages, hence the need
+#  to test each of them separately, and prohibit executing a scenario where
+#  either of them can happen nondeterministically.
+#
 # [--let $mts_spco_gd_trx_blocking_client = <STATEMENT>]
 #  The statement to be executed by a client connection that should be
 #  blocked by one or both workers waiting on the commit queue.
@@ -67,7 +81,7 @@
 #   6. On the replica, ensure that client connection B is waiting on the
 #      lock being held by Worker 2 or Worker 3, reaching the state
 #      $mts_spco_gd_state_blocking_client.
-#   7. On the replica, using client conneciton A, rollback the pending
+#   7. On the replica, using client connection A, rollback the pending
 #      transaction, leading to the following lock acquisition dependencies:
 #      Client B --statement required lock--> Worker 2 --commit order
 #      lock--> Worker 1 --statement required lock--> Client B.
@@ -97,6 +111,10 @@ if ($mts_spco_gd_error_expected_replica == '')
 {
   --let $mts_spco_gd_error_expected_replica = 0
 }
+if ($mts_spco_gd_worker_3_only_runs_after_deadlock == '')
+{
+  --let $mts_spco_gd_worker_3_only_runs_after_deadlock = 0
+}
 
 --source include/rpl_connection_slave.inc
 --let $mts_spco_gd_is_replica_sql_running = query_get_value(SHOW REPLICA STATUS, Replica_SQL_Running, 1)
@@ -111,7 +129,7 @@ if ($mts_spco_gd_seqno_to_wait_for == '')
 }
 
 --inc $mts_spco_gd_seqno_to_wait_for
---let $mts_spco_gd_gtid_to_wait_for = "aaaaaaaa-1111-bbbb-2222-cccccccccccc:$mts_spco_gd_seqno_to_wait_for"
+--let $mts_spco_gd_gtid_to_wait_for_1 = "aaaaaaaa-1111-bbbb-2222-cccccccccccc:$mts_spco_gd_seqno_to_wait_for"
 
 #   1. Execute $mts_spco_gd_trx_blocking_worker_1,
 #      $mts_spco_gd_trx_assigned_worker_2 and
@@ -121,27 +139,52 @@ if ($mts_spco_gd_seqno_to_wait_for == '')
 --source include/rpl_connection_master.inc
 --let $debug_point = set_commit_parent_100
 --source include/add_debug_point.inc
---eval SET GTID_NEXT = $mts_spco_gd_gtid_to_wait_for
---eval $mts_spco_gd_trx_blocking_worker_1
 
+--eval SET GTID_NEXT = $mts_spco_gd_gtid_to_wait_for_1
+--eval $mts_spco_gd_trx_blocking_worker_1
 SET GTID_NEXT = AUTOMATIC;
+
 if ($mts_spco_gd_trx_assigned_worker_2 != '')
 {
+  --inc $mts_spco_gd_seqno_to_wait_for
+  --let $mts_spco_gd_gtid_to_wait_for_2 = "aaaaaaaa-1111-bbbb-2222-cccccccccccc:$mts_spco_gd_seqno_to_wait_for"
+  --eval SET GTID_NEXT = $mts_spco_gd_gtid_to_wait_for_2
   --eval $mts_spco_gd_trx_assigned_worker_2
+  SET GTID_NEXT = AUTOMATIC;
 }
 if ($mts_spco_gd_trx_assigned_worker_3 != '')
 {
+  --inc $mts_spco_gd_seqno_to_wait_for
+  --let $mts_spco_gd_gtid_to_wait_for_3 = "aaaaaaaa-1111-bbbb-2222-cccccccccccc:$mts_spco_gd_seqno_to_wait_for"
+  --eval SET GTID_NEXT = $mts_spco_gd_gtid_to_wait_for_3
   --eval $mts_spco_gd_trx_assigned_worker_3
+  SET GTID_NEXT = AUTOMATIC;
 }
+
 --source include/remove_debug_point.inc
 
 #   2. On the replica, using client connection A, start a transaction and
 #      assign it the same GTID as to one of the statements issued on the
 #      source.
 --source include/rpl_connection_slave.inc
---eval SET GTID_NEXT = $mts_spco_gd_gtid_to_wait_for
+--eval SET GTID_NEXT = $mts_spco_gd_gtid_to_wait_for_1
 BEGIN;
 
+if ($mts_spco_gd_trx_assigned_worker_2 != '')
+{
+  --let $rpl_connection_name = rpl_slave_connection_2
+  --source include/rpl_connection.inc
+  --eval SET GTID_NEXT = $mts_spco_gd_gtid_to_wait_for_2
+  BEGIN;
+}
+if ($mts_spco_gd_trx_assigned_worker_3 != '')
+{
+  --let $rpl_connection_name = rpl_slave_connection_3
+  --source include/rpl_connection.inc
+  --eval SET GTID_NEXT = $mts_spco_gd_gtid_to_wait_for_3
+  BEGIN;
+}
+
 --source include/rpl_connection_slave1.inc
 #   3. Start the replication threads
 --source include/start_slave_sql.inc
@@ -166,20 +209,29 @@ BEGIN;
 
 #   4. On the replica, ensure that the applier worker threads are waiting
 #      on the pending client connection transaction.
---let $mts_spco_gd_pending_workers = 0
 if ($mts_spco_gd_trx_assigned_worker_2 != '')
 {
-  --inc $mts_spco_gd_pending_workers
+  --let $rpl_connection_name = rpl_slave_connection_2
+  --source include/rpl_connection.inc
+  ROLLBACK;
+  SET GTID_NEXT = AUTOMATIC;
+  --echo include/wait_condition.inc [First worker must wait on commit order]
+  --let $wait_condition = SELECT count(*) = 1 FROM information_schema.processlist WHERE STATE = "Waiting for preceding transaction to commit"
+  --source include/wait_condition.inc
 }
 if ($mts_spco_gd_trx_assigned_worker_3 != '')
 {
-  --inc $mts_spco_gd_pending_workers
-}
-if ($mts_spco_gd_pending_workers != 0)
-{
-  --echo include/wait_condition.inc [Workers must wait on commit order]
-  --let $wait_condition = SELECT count(*) = $mts_spco_gd_pending_workers FROM information_schema.processlist WHERE STATE = "Waiting for preceding transaction to commit"
-  --source include/wait_condition.inc
+  if ($mts_spco_gd_worker_3_only_runs_after_deadlock == 0)
+  {
+    --source include/wait_condition.inc
+    --let $rpl_connection_name = rpl_slave_connection_3
+    --source include/rpl_connection.inc
+    ROLLBACK;
+    SET GTID_NEXT = AUTOMATIC;
+    --echo include/wait_condition.inc [Second worker must wait on commit order]
+    --let $wait_condition = SELECT count(*) = 2 FROM information_schema.processlist WHERE STATE = "Waiting for preceding transaction to commit"
+    --source include/wait_condition.inc
+  }
 }
 
 #   5. On the replica, using client connection B, execute
@@ -189,6 +241,7 @@ if ($mts_spco_gd_pending_workers != 0)
 #      Worker 2 --commit order lock--> Worker 1 --gtid lock--> Client A.
 if ($mts_spco_gd_trx_blocking_client != '')
 {
+  --source include/rpl_connection_slave1.inc
   --send_eval $mts_spco_gd_trx_blocking_client
 }
 
@@ -209,7 +262,7 @@ if ($mts_spco_gd_state_blocking_client != '')
   --source include/wait_condition.inc
 }
 
-#   7. On the replica, using client conneciton A, rollback the pending
+#   7. On the replica, using client connection A, rollback the pending
 #      transaction, leading to the following lock acquisition dependencies:
 #      Client B --statement required lock--> Worker 2 --commit order
 #      lock--> Worker 1 --statement required lock--> Client B.
@@ -240,6 +293,20 @@ if ($mts_spco_gd_wait_for_coordinator_running_state != '')
   }
 }
 
+if ($mts_spco_gd_trx_assigned_worker_3 != '')
+{
+  if ($mts_spco_gd_worker_3_only_runs_after_deadlock == 1)
+  {
+    --echo include/wait_condition.inc [Coordinator must wait for workers to stop]
+    --let $wait_condition = SELECT count(*) = 1 FROM information_schema.processlist WHERE STATE = "Waiting for workers to exit"
+    --source include/wait_condition.inc
+    --let $rpl_connection_name = rpl_slave_connection_3
+    --source include/rpl_connection.inc
+    ROLLBACK;
+    SET GTID_NEXT = AUTOMATIC;
+  }
+}
+
 #  10. Wait for the replica applier thread to error out with
 #      $mts_spco_gd_error_expected_replica, if any is defined.
 if ($mts_spco_gd_error_expected_replica != 0)
@@ -280,6 +347,7 @@ if ($mts_spco_gd_error_expected_replica != 0)
 --let $mts_spco_gd_trx_blocking_worker_1 =
 --let $mts_spco_gd_trx_assigned_worker_2 =
 --let $mts_spco_gd_trx_assigned_worker_3 =
+--let $mts_spco_gd_worker_3_only_runs_after_deadlock =
 --let $mts_spco_gd_trx_blocking_client =
 --let $mts_spco_gd_state_blocking_client =
 --let $mts_spco_gd_wait_for_coordinator_running_state =

mysql-test/extra/rpl_tests/mts_spco_generate_deadlock_setup.inc

@@ -30,3 +30,10 @@ SET GLOBAL slave_preserve_commit_order = ON;
 --eval SET GLOBAL slave_transaction_retries = $mts_spco_gd_transaction_retries
 --replace_result $mts_spco_gd_innodb_wait_timeout INNODB_LOCK_WAIT_TIMEOUT
 --eval SET GLOBAL innodb_lock_wait_timeout = $mts_spco_gd_innodb_wait_timeout
+
+--let $rpl_connection_name = rpl_slave_connection_2
+--let $rpl_server_number = 2
+--source include/rpl_connect.inc
+--let $rpl_connection_name = rpl_slave_connection_3
+--let $rpl_server_number = 2
+--source include/rpl_connect.inc