Bug#26024253: GR MEMBER STOP MUST STOP ALL ASYNCHRONOUS CHANNELS THAT ARE FEEDING DATA INTO IT

Hemant Dangi · Hemant Dangi · commit a751a0bfbc7e · 2017-08-24T19:59:47.000+02:00
Issue
=====
If a member that has asynchronous channels feeding data into it - either on
multi-primary deploy or the primary on single-primary deploy - does stop, all
asynchronous channels must be stopped to avoid that changes are made locally
(not replicated to the group).

Even when super_read_only it is set on STOP GROUP_REPLICATION, that will not
affect already running asynchronous which can still do changes.

Solution
========
When group replication stops, due to error or due to executing stop
group_replication command, all asynchronous channels are stopped.
diff --git a/rapid/plugin/group_replication/include/gcs_event_handlers.h b/rapid/plugin/group_replication/include/gcs_event_handlers.h
@@ -67,7 +67,8 @@ class Plugin_gcs_events_handler: public Gcs_communication_event_listener,
   Plugin_gcs_events_handler(Applier_module_interface* applier_module,
                             Recovery_module* recovery_module,
                             Plugin_gcs_view_modification_notifier* vc_notifier,
-                            Compatibility_module* compatibility_manager);
+                            Compatibility_module* compatibility_manager,
+                            ulong components_stop_timeout);
   virtual ~Plugin_gcs_events_handler();
 
   /*
@@ -80,6 +81,14 @@ class Plugin_gcs_events_handler: public Gcs_communication_event_listener,
   void on_suspicions(const std::vector<Gcs_member_identifier>& members,
                      const std::vector<Gcs_member_identifier>& unreachable) const;
 
+  /**
+    Sets the component stop timeout.
+
+    @param[in]  timeout      the timeout
+  */
+  void set_stop_wait_timeout (ulong timeout){
+    stop_wait_timeout= timeout;
+  }
 
 private:
   /*
@@ -275,6 +284,9 @@ class Plugin_gcs_events_handler: public Gcs_communication_event_listener,
   /**The status of this member when it joins*/
   st_compatibility_types* joiner_compatibility_status;
 
+  /* Component stop timeout on shutdown */
+  ulong stop_wait_timeout;
+
 #ifndef DBUG_OFF
   bool set_number_of_members_on_view_changed_to_10;
 #endif
diff --git a/rapid/plugin/group_replication/src/applier.cc b/rapid/plugin/group_replication/src/applier.cc
@@ -680,6 +680,16 @@ void Applier_module::leave_group_on_failure()
   bool set_read_mode= false;
   Gcs_operations::enum_leave_state state= gcs_module->leave();
 
+  int error= channel_stop_all(CHANNEL_APPLIER_THREAD|CHANNEL_RECEIVER_THREAD,
+                              stop_wait_timeout);
+  if (error)
+  {
+    log_message(MY_ERROR_LEVEL,
+                "Error stopping all replication channels while server was"
+                " leaving the group. Please check the error log for additional"
+                " details. Got error: %d", error);
+  }
+
   std::stringstream ss;
   plugin_log_level log_severity= MY_WARNING_LEVEL;
   switch (state)
diff --git a/rapid/plugin/group_replication/src/gcs_event_handlers.cc b/rapid/plugin/group_replication/src/gcs_event_handlers.cc
@@ -29,10 +29,12 @@ Plugin_gcs_events_handler::
 Plugin_gcs_events_handler(Applier_module_interface* applier_module,
                           Recovery_module* recovery_module,
                           Plugin_gcs_view_modification_notifier* vc_notifier,
-                          Compatibility_module* compatibility_module)
+                          Compatibility_module* compatibility_module,
+                          ulong components_stop_timeout)
 : applier_module(applier_module), recovery_module(recovery_module),
   view_change_notifier(vc_notifier),
-  compatibility_manager(compatibility_module)
+  compatibility_manager(compatibility_module),
+  stop_wait_timeout(components_stop_timeout)
 {
   this->temporary_states= new std::set<Group_member_info*,
                                        Group_member_info_pointer_comparator>();
@@ -1530,6 +1532,16 @@ void
 Plugin_gcs_events_handler::leave_group_on_error() const
 {
   Gcs_operations::enum_leave_state state= gcs_module->leave();
+  int error= channel_stop_all(CHANNEL_APPLIER_THREAD|CHANNEL_RECEIVER_THREAD,
+                              stop_wait_timeout);
+  if (error)
+  {
+    log_message(MY_ERROR_LEVEL,
+                "Error stopping all replication channels while server was"
+                " leaving the group. Please check the error log for additional"
+                " details. Got error: %d", error);
+  }
+
   std::stringstream ss;
   plugin_log_level log_severity= MY_WARNING_LEVEL;
   switch (state)
diff --git a/rapid/plugin/group_replication/src/plugin.cc b/rapid/plugin/group_replication/src/plugin.cc
@@ -234,7 +234,7 @@ void initialize_group_partition_handler();
 int start_group_communication();
 void declare_plugin_running();
 int leave_group();
-int terminate_plugin_modules();
+int terminate_plugin_modules(bool flag_stop_async_channel= false);
 int terminate_applier_module();
 int terminate_recovery_module();
 void terminate_asynchronous_channels_observer();
@@ -813,7 +813,7 @@ int plugin_group_replication_stop()
   /* first leave all joined groups (currently one) */
   leave_group();
 
-  int error= terminate_plugin_modules();
+  int error= terminate_plugin_modules(true);
 
   group_replication_running= false;
   shared_plugin_stop_lock->release_write_lock();
@@ -836,7 +836,7 @@ int plugin_group_replication_stop()
   DBUG_RETURN(error);
 }
 
-int terminate_plugin_modules()
+int terminate_plugin_modules(bool flag_stop_async_channel)
 {
 
   if(terminate_recovery_module())
@@ -867,6 +867,23 @@ int terminate_plugin_modules()
 
   terminate_asynchronous_channels_observer();
 
+  if (flag_stop_async_channel)
+  {
+    int channel_err= channel_stop_all(CHANNEL_APPLIER_THREAD|CHANNEL_RECEIVER_THREAD,
+                            components_stop_timeout_var);
+    if (channel_err)
+    {
+      log_message(MY_ERROR_LEVEL,
+                  "Error stopping all replication channels while server was"
+                  " leaving the group. Please check the error log for "
+                  "additional details. Got error: %d", channel_err);
+      if (!error)
+      {
+        error= GROUP_REPLICATION_CONFIGURATION_ERROR;
+      }
+    }
+  }
+
   delete group_partition_handler;
   group_partition_handler= NULL;
 
@@ -1357,7 +1374,8 @@ int start_group_communication()
   events_handler= new Plugin_gcs_events_handler(applier_module,
                                                 recovery_module,
                                                 view_change_notifier,
-                                                compatibility_mgr);
+                                                compatibility_mgr,
+                                                components_stop_timeout_var);
 
   view_change_notifier->start_view_modification();
 
@@ -1870,6 +1888,10 @@ static void update_component_timeout(MYSQL_THD thd, SYS_VAR *var,
   {
     recovery_module->set_stop_wait_timeout(in_val);
   }
+  if (events_handler != NULL)
+  {
+    events_handler->set_stop_wait_timeout(in_val);
+  }
 
   DBUG_VOID_RETURN;
 }
diff --git a/rapid/plugin/group_replication/src/recovery.cc b/rapid/plugin/group_replication/src/recovery.cc
@@ -1,4 +1,4 @@
-/* Copyright (c) 2014, 2016, Oracle and/or its affiliates. All rights reserved.
+/* Copyright (c) 2014, 2017, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -187,6 +187,16 @@ void Recovery_module::leave_group_on_recovery_failure()
 
   Gcs_operations::enum_leave_state state= gcs_module->leave();
 
+  int error= channel_stop_all(CHANNEL_APPLIER_THREAD|CHANNEL_RECEIVER_THREAD,
+                              stop_wait_timeout);
+  if (error)
+  {
+    log_message(MY_ERROR_LEVEL,
+                "Error stopping all replication channels while server was"
+                " leaving the group. Please check the error log for additional"
+                " details. Got error: %d", error);
+  }
+
   std::stringstream ss;
   plugin_log_level log_severity= MY_WARNING_LEVEL;
   switch (state)
diff --git a/rapid/plugin/group_replication/tests/mtr/r/gr_stop_async_on_stop_gr.result b/rapid/plugin/group_replication/tests/mtr/r/gr_stop_async_on_stop_gr.result
@@ -0,0 +1,76 @@
+include/group_replication.inc
+Warnings:
+Note	####	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	####	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+[connection server1]
+
+############################################################
+#  1. Setup Group Replication on server 1.
+[connection server1]
+include/start_and_bootstrap_group_replication.inc
+
+############################################################
+#  2. Setup a asynchronous replication connection from server 2
+#     into group (server 1)
+[connection server1]
+CHANGE MASTER TO MASTER_HOST='localhost', MASTER_USER='root', MASTER_PORT=SERVER_2_PORT, MASTER_AUTO_POSITION=1 for channel 'ch2_1';
+Warnings:
+Note	1759	Sending passwords in plain text without SSL/TLS is extremely insecure.
+Note	1760	Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information.
+include/start_slave.inc
+
+############################################################
+#  3. Execute some transactions on server 2.
+[connection server2]
+CREATE TABLE t1 (c1 INT NOT NULL PRIMARY KEY) ENGINE=InnoDB;
+INSERT INTO t1 VALUES (0);
+
+############################################################
+#  4. Wait until transactions executed on server 2 are applied
+#     on group.
+include/sync_slave_sql_with_master.inc
+
+############################################################
+#  5. Activate group_replication_stop_all_channels_failure
+#     debug sync point, which will return
+#     ER_GROUP_REPLICATION_CONFIGURATION error, when stop
+#     group replication calls to stop all replication channels.
+[connection server1]
+# 5.1. Stop Group Replication
+SET @debug_save= @@GLOBAL.DEBUG;
+SET @@GLOBAL.DEBUG='d,group_replication_stop_all_channels_failure';
+STOP GROUP_REPLICATION;
+ERROR HY000: The server is not configured properly to be an active member of the group. Please see more details on error log.
+SET @@GLOBAL.DEBUG= @debug_save;
+# 5.2. Verify member is OFFLINE
+include/gr_wait_for_member_state.inc
+# 5.3. Verify occurrence of error message
+Occurrences of 'Error stopping all replication channels while server was leaving the group. Please check the error log for additional details.' in the input file: 1
+Occurrences of 'Error stopping channel: ch2_1. Got error: 1, Error_code: 1' in the input file: 1
+Occurrences of 'Error stopping channel: . Got error: 1, Error_code: 1' in the input file: 1
+
+############################################################
+#  6. Execute data on server 2 and ensure it is not accepted on server 1.
+[connection server2]
+INSERT INTO t1 VALUES (1);
+
+############################################################
+#  7. Verify channel ch2_1 IO and SQL THREADS are OFF
+[connection server1]
+include/wait_for_slave_to_stop.inc
+include/assert.inc [Verify channel ch2_1 IO_THREAD is OFF]
+include/assert.inc [Verify channel ch2_1 SQL_THREAD is OFF]
+
+############################################################
+#  8. Verify data isn't replicated to group i.e. server1
+include/assert.inc ['There is no value 1 in table t1']
+
+############################################################
+#  9. Clean Up
+[connection server1]
+SET GLOBAL read_only= 0;
+RESET SLAVE ALL FOR CHANNEL 'ch2_1';
+DROP TABLE test.t1;
+[connection server2]
+DROP TABLE test.t1;
+include/group_replication_end.inc
diff --git a/rapid/plugin/group_replication/tests/mtr/t/gr_stop_async_on_stop_gr-master.opt b/rapid/plugin/group_replication/tests/mtr/t/gr_stop_async_on_stop_gr-master.opt
@@ -0,0 +1 @@
+--no-console --log_error=$MYSQLTEST_VARDIR/tmp/gr_stop_async_on_stop_gr.err
diff --git a/rapid/plugin/group_replication/tests/mtr/t/gr_stop_async_on_stop_gr.test b/rapid/plugin/group_replication/tests/mtr/t/gr_stop_async_on_stop_gr.test
diff --git a/sql/rpl_channel_service_interface.cc b/sql/rpl_channel_service_interface.cc
diff --git a/sql/rpl_channel_service_interface.h b/sql/rpl_channel_service_interface.h

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+--no-console --log_error=$MYSQLTEST_VARDIR/tmp/gr_stop_async_on_stop_gr.err`