Bug#30627292 NDB METADATA SYNC FAILS TO SYNC TABLES BELONGING TO DETECTED SCHEMA

Problem:
--------

The ndb_metadata_sync system variable is set by the user to trigger
synchronization of metadata between the DD of a MySQL server and NDB
Dictionary. It is designed to detect and synchronize mismatches in terms of
logfile groups, tablespaces, schemata, and tables.

When the user triggers a sync using the above variable, the change monitor
thread enters a phase of continuous detection submitting objects to the binlog
thread for synchronization. It eventually stalls itself when it determines
that there are no more mismatches and waits until the binlog thread has made
its way through the queue of objects to be synchronized. The ndb_metadata_sync
variable was flipped when all objects in the afore-mentioned queue were
processed.

One scenario involving the detection of schema not present in the DD but
currently being used in the NDB Dictionary sometimes led to the variable
being flipped before the tables belonging to the schema were successfully
synchronized. This is due to the fact that tables belonging to "detected"
schema can only be detected as mismatches themselves when the schema
is created in the DD of the MySQL server. The timing of this synchronization
by the binlog thread is asynchronous by design.

Fix:
----

The fix is to essentially delay the flipping of the ndb_metadata_sync
variable. After the first instance of all objects in the queue being
processed is detected, the change monitor thread embarks on at least one
more detection and synchronization cycle. These cycles are repeated until
it is ascertained that the entire queue has been processed for a second
time.

Change-Id: Ib2395f84c5516c003d57a2339591352b15ce3a2e

Author: arnabray21

mysql-test/suite/ndb_ddl/metadata_immediate_sync.result

@@ -156,3 +156,112 @@ DROP DATAFILE 'ts2_datafile.dat';
 DROP TABLESPACE ts2;
 DROP LOGFILE GROUP lg1
 ENGINE NDB;
+CREATE DATABASE db1;
+USE db1;
+CREATE TABLE t1_ndb (
+a INT PRIMARY KEY,
+b INT
+) ENGINE NDB;
+CREATE TABLE t2_ndb (
+a INT PRIMARY KEY,
+b VARCHAR(255)
+) ENGINE NDB;
+CREATE DATABASE db2;
+USE db2;
+CREATE TABLE t1_ndb (
+a INT PRIMARY KEY,
+b INT
+) ENGINE NDB;
+CREATE TABLE t2_ndb (
+a INT PRIMARY KEY,
+b VARCHAR(255)
+) ENGINE NDB;
+CREATE DATABASE db3;
+USE db3;
+CREATE TABLE t1_innodb (
+a INT PRIMARY KEY,
+b INT
+);
+CREATE TABLE t2_innodb (
+a INT PRIMARY KEY,
+b VARCHAR(255)
+);
+CREATE DATABASE db4;
+USE db4;
+CREATE TABLE t1_ndb (
+a INT PRIMARY KEY,
+b INT
+) ENGINE NDB;
+CREATE TABLE t2_ndb (
+a INT PRIMARY KEY,
+b VARCHAR(255)
+) ENGINE NDB;
+CREATE TABLE t1_innodb (
+a INT PRIMARY KEY,
+b INT
+);
+CREATE TABLE t2_innodb (
+a INT PRIMARY KEY,
+b VARCHAR(255)
+);
+DROP DATABASE db1;
+DROP DATABASE db4;
+SET GLOBAL ndb_metadata_sync = true;
+SHOW DATABASES LIKE 'db%';
+Database (db%)
+db1
+db2
+db3
+db4
+USE db1;
+SHOW TABLES;
+Tables_in_db1
+t1_ndb
+t2_ndb
+SHOW CREATE TABLE t1_ndb;
+Table	Create Table
+t1_ndb	CREATE TABLE `t1_ndb` (
+  `a` int NOT NULL,
+  `b` int DEFAULT NULL,
+  PRIMARY KEY (`a`)
+) ENGINE=ndbcluster DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='NDB_TABLE=READ_BACKUP=1'
+SHOW CREATE TABLE t2_ndb;
+Table	Create Table
+t2_ndb	CREATE TABLE `t2_ndb` (
+  `a` int NOT NULL,
+  `b` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`a`)
+) ENGINE=ndbcluster DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='NDB_TABLE=READ_BACKUP=1'
+USE db2;
+SHOW TABLES;
+Tables_in_db2
+t1_ndb
+t2_ndb
+USE db3;
+SHOW TABLES;
+Tables_in_db3
+t1_innodb
+t2_innodb
+USE db4;
+SHOW TABLES;
+Tables_in_db4
+t1_ndb
+t2_ndb
+SHOW CREATE TABLE t1_ndb;
+Table	Create Table
+t1_ndb	CREATE TABLE `t1_ndb` (
+  `a` int NOT NULL,
+  `b` int DEFAULT NULL,
+  PRIMARY KEY (`a`)
+) ENGINE=ndbcluster DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='NDB_TABLE=READ_BACKUP=1'
+SHOW CREATE TABLE t2_ndb;
+Table	Create Table
+t2_ndb	CREATE TABLE `t2_ndb` (
+  `a` int NOT NULL,
+  `b` varchar(255) DEFAULT NULL,
+  PRIMARY KEY (`a`)
+) ENGINE=ndbcluster DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci COMMENT='NDB_TABLE=READ_BACKUP=1'
+DROP DATABASE db1;
+DROP DATABASE db2;
+DROP DATABASE db3;
+DROP DATABASE db4;

mysql-test/suite/ndb_ddl/metadata_immediate_sync.test

@@ -2,7 +2,7 @@
 --source suite/ndb/include/backup_restore_setup.inc
 
 #
-# Basic schema detection + synchronization test using
+# Case 1: Basic schema detection + synchronization test using
 # the ndb_metadata_sync option. This is a duplicate of
 # case 1 in the metadata_sync test.
 # - Cause mismatch between NDB Dictionary and DD
@@ -171,5 +171,116 @@ ALTER TABLESPACE ts2
 DROP TABLESPACE ts2;
 DROP LOGFILE GROUP lg1
   ENGINE NDB;
+
+#
+# Case 2: Automatic synchronization of schemata.
+# This is a duplicate of case 2 in the metadata_sync test
+# - Cause mismatch in terms of schema between NDB Dictionary and DD
+# - Check if the schema mismatch is detected
+# - Check if the schema mismatch is synchronized
+#
+
+# db1 - Only NDB tables
+CREATE DATABASE db1;
+USE db1;
+CREATE TABLE t1_ndb (
+  a INT PRIMARY KEY,
+  b INT
+) ENGINE NDB;
+CREATE TABLE t2_ndb (
+  a INT PRIMARY KEY,
+  b VARCHAR(255)
+) ENGINE NDB;
+
+# db2 - Only NDB tables
+CREATE DATABASE db2;
+USE db2;
+CREATE TABLE t1_ndb (
+  a INT PRIMARY KEY,
+  b INT
+) ENGINE NDB;
+CREATE TABLE t2_ndb (
+  a INT PRIMARY KEY,
+  b VARCHAR(255)
+) ENGINE NDB;
+
+# db3 - Only InnoDB tables
+CREATE DATABASE db3;
+USE db3;
+CREATE TABLE t1_innodb (
+  a INT PRIMARY KEY,
+  b INT
+);
+CREATE TABLE t2_innodb (
+  a INT PRIMARY KEY,
+  b VARCHAR(255)
+);
+
+# db4 - Both NDB and InnoDB tables
+CREATE DATABASE db4;
+USE db4;
+CREATE TABLE t1_ndb (
+  a INT PRIMARY KEY,
+  b INT
+) ENGINE NDB;
+CREATE TABLE t2_ndb (
+  a INT PRIMARY KEY,
+  b VARCHAR(255)
+) ENGINE NDB;
+CREATE TABLE t1_innodb (
+  a INT PRIMARY KEY,
+  b INT
+);
+CREATE TABLE t2_innodb (
+  a INT PRIMARY KEY,
+  b VARCHAR(255)
+);
+
+# backup
+--disable_query_log
+--source include/ndb_backup.inc
+--enable_query_log
+
+# Cause mismatch by dropping db1 and db4 and restoring metadata
+DROP DATABASE db1;
+DROP DATABASE db4;
+
+--exec $NDB_RESTORE -b $the_backup_id -n 1 -m --exclude-databases=db2 $NDB_BACKUPS-$the_backup_id >> $NDB_TOOLS_OUTPUT
+
+#
+# At this point the state of the schemata is as follows:
+# - db1: Used in NDB Dictionary, non-existent in DD
+# - db2: Used in NDB Dictionary, exists in DD
+# - db3: Not used in NDB Dictionary, exists in DD
+# - db4: Used in NDB Dictionary, non-existent in DD
+#
+# The auto sync mechanism will create db1 and db4 in the DD.
+# It will also synchronize all NDB tables contained in db1 and db4
+#
+
+# Wait until the changes have been synced
+--let $max_wait = 60
+--source wait_immediate_metadata_sync.inc
+
+# Check if the objects have been synchronized as expected
+SHOW DATABASES LIKE 'db%';
+USE db1;
+SHOW TABLES;
+SHOW CREATE TABLE t1_ndb;
+SHOW CREATE TABLE t2_ndb;
+USE db2;
+SHOW TABLES;
+USE db3;
+SHOW TABLES;
+USE db4;
+SHOW TABLES;
+SHOW CREATE TABLE t1_ndb;
+SHOW CREATE TABLE t2_ndb;
+
+# Clean up
+DROP DATABASE db1;
+DROP DATABASE db2;
+DROP DATABASE db3;
+DROP DATABASE db4;
 --source suite/ndb/include/backup_restore_cleanup.inc
 --remove_file $NDB_TOOLS_OUTPUT

storage/ndb/plugin/ndb_metadata_change_monitor.cc

@@ -46,7 +46,7 @@
 #include "storage/ndb/plugin/ndb_thd_ndb.h"               // Thd_ndb
 
 Ndb_metadata_change_monitor::Ndb_metadata_change_monitor()
-    : Ndb_component("Metadata") {}
+    : Ndb_component("Metadata"), m_mark_sync_complete{false} {}
 
 Ndb_metadata_change_monitor::~Ndb_metadata_change_monitor() {}
 
@@ -612,18 +612,36 @@ void Ndb_metadata_change_monitor::do_run() {
       log_verbose(10, "Metadata check completed");
 
       if (controller.get_metadata_sync() && controller.all_changes_detected()) {
-        log_info("Metadata detection complete");
         // All changes at this point in time have been detected. Since the
         // ndb_metadata_sync option has been set, we don't expect more changes.
         // Stall the thread and prevent it from checking for further mismatches
         // until the current queue has been synchronized by the binlog thread
         mysql_mutex_lock(&m_sync_done_mutex);
         mysql_cond_wait(&m_sync_done_cond, &m_sync_done_mutex);
         mysql_mutex_unlock(&m_sync_done_mutex);
-        log_info("Metadata synchronization complete");
-        // Set ndb_metadata_sync to false to denote that all changes have been
-        // detected and synchronized
-        opt_ndb_metadata_sync = false;
+        if (!m_mark_sync_complete) {
+          // This is the first instance of the binlog thread having synchronized
+          // all changes submitted to it. However, the change monitor thread
+          // has been stalled for a while so we opt for at least one more
+          // detection and sync cycle to ensure that all changes are synced.
+          // This is particularly relevant to synchronization of schema objects
+          // since they have to be installed in DD for their tables to be
+          // detected. This synchronization is dependent on the load on the
+          // binlog thread so an additional detection and sync run after we know
+          // for a fact that such schemata have been installed could be useful.
+          //
+          // The below flag denotes that the we've already detected an instance
+          // of all objects having been synchronized and that ndb_metadata_sync
+          // can be flipped the next time we detect the same condition
+          m_mark_sync_complete = true;
+        } else {
+          log_info("Metadata synchronization complete");
+          // Set ndb_metadata_sync to false to denote that all changes have been
+          // detected and synchronized
+          opt_ndb_metadata_sync = false;
+          // Reset the flag to its default value
+          m_mark_sync_complete = false;
+        }
       }
     }
   }

storage/ndb/plugin/ndb_metadata_change_monitor.h

@@ -43,6 +43,7 @@ class Ndb_metadata_change_monitor : public Ndb_component {
   mysql_cond_t m_wait_cond;
   static mysql_mutex_t m_sync_done_mutex;  // protects m_sync_done_cond
   static mysql_cond_t m_sync_done_cond;
+  bool m_mark_sync_complete;
 
  public:
   Ndb_metadata_change_monitor();