Bug#35840020 Non distribution aware node selection is imbalanced

Unhinted transactions, or hintable transactions where hinting
is disabled will use a round-robin (RR) algorithm, balancing
the transaction load across a set of candidate data nodes.

The ndb_optimized_node_selection option controls which nodes
are candidates.

                        ndb_optimized_node_selection & 1
  All data nodes        0
  Local data nodes      1

Locality is defined or inferred by a number of NdbApi
level concepts such as :
  Config   : LocationDomain
  Api      : DataNodeNeighbour
  Internal : tryBind remote address->Group
  Config   : Group

In both cases there can be multiple candidates, so NdbApi
attempts to spread the load across the candidates.

However the state memory required to do this is part of the
Ndb (session) object, and is reset (to the same 'next' position)
in every new Ndb object.

In some use cases new Ndb objects are always used, such as
when a SQL user connects, issues an unhinted query and then
disconnects.

In this case there is no RR balance, resulting in resource
usage imbalances and limiting system scalability and capacity.

This is fixed so that in these cases the first transaction
issued by a connection is also handled in a round-robin
way, avoiding persistent imbalance.

The MTR testcase ndb_optimized_node_selection is extended
to cover this scenario.

Change-Id: Ia491b3bc4ae96e5e57a27baa375d2d84b3068d60

Author: frazerclement

mysql-test/suite/ndb/r/ndb_optimized_node_selection.result

@@ -47,6 +47,10 @@ READ PRIMARY o-n-s 0 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+READ PRIMARY o-n-s 0 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 set ndb_optimized_node_selection=1;
 show variables like 'ndb_optimized_node_selection';
 Variable_name	Value
@@ -63,6 +67,10 @@ READ PRIMARY o-n-s 1 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+READ PRIMARY o-n-s 1 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 set ndb_optimized_node_selection=2;
 show variables like 'ndb_optimized_node_selection';
 Variable_name	Value
@@ -79,6 +87,10 @@ READ PRIMARY o-n-s 2 case Autocommit update by pk
 node_id	balance
 1	NONE
 2	ALL
+READ PRIMARY o-n-s 2 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 set ndb_optimized_node_selection=3;
 show variables like 'ndb_optimized_node_selection';
 Variable_name	Value
@@ -95,6 +107,10 @@ READ PRIMARY o-n-s 3 case Autocommit update by pk
 node_id	balance
 1	NONE
 2	ALL
+READ PRIMARY o-n-s 3 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 drop table basecounts;
 drop table t1;
 create table t1 (a int primary key, b int) engine=ndb comment="NDB_TABLE=READ_BACKUP=1";
@@ -116,6 +132,10 @@ READ BACKUP o-n-s 0 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+READ BACKUP o-n-s 0 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 set ndb_optimized_node_selection=1;
 show variables like 'ndb_optimized_node_selection';
 Variable_name	Value
@@ -132,6 +152,10 @@ READ BACKUP o-n-s 1 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+READ BACKUP o-n-s 1 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 set ndb_optimized_node_selection=2;
 show variables like 'ndb_optimized_node_selection';
 Variable_name	Value
@@ -148,6 +172,10 @@ READ BACKUP o-n-s 2 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+READ BACKUP o-n-s 2 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 set ndb_optimized_node_selection=3;
 show variables like 'ndb_optimized_node_selection';
 Variable_name	Value
@@ -164,6 +192,10 @@ READ BACKUP o-n-s 3 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+READ BACKUP o-n-s 3 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 drop table basecounts;
 drop table t1;
 create table t1 (a int primary key, b int) engine=ndb comment="NDB_TABLE=FULLY_REPLICATED=1";
@@ -185,6 +217,10 @@ FULLY REPLICATED o-n-s 0 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+FULLY REPLICATED o-n-s 0 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 set ndb_optimized_node_selection=1;
 show variables like 'ndb_optimized_node_selection';
 Variable_name	Value
@@ -201,6 +237,10 @@ FULLY REPLICATED o-n-s 1 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+FULLY REPLICATED o-n-s 1 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 set ndb_optimized_node_selection=2;
 show variables like 'ndb_optimized_node_selection';
 Variable_name	Value
@@ -217,6 +257,10 @@ FULLY REPLICATED o-n-s 2 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+FULLY REPLICATED o-n-s 2 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 set ndb_optimized_node_selection=3;
 show variables like 'ndb_optimized_node_selection';
 Variable_name	Value
@@ -233,5 +277,9 @@ FULLY REPLICATED o-n-s 3 case Autocommit update by pk
 node_id	balance
 1	HALF
 2	HALF
+FULLY REPLICATED o-n-s 3 case Unhinted table scan on new connection
+node_id	balance
+1	HALF
+2	HALF
 drop table basecounts;
 drop table t1;

mysql-test/suite/ndb/t/ndb_optimized_node_selection.test

@@ -64,7 +64,7 @@ while ($dist < 3)
     show variables like 'ndb_optimized_node_selection';
 
     --let $case=0
-    while ($case < 3)
+    while ($case < 4)
     {
       --disable_query_log
       --disable_result_log
@@ -78,7 +78,8 @@ while ($dist < 3)
            where counter_name="TRANSACTIONS"
         group by node_id;
 
-      --let $iter=10
+      --let $iter=100
+      --let $errbound=$iter/10
       --let $rpt=$iter
       while ($rpt)
       {
@@ -100,6 +101,19 @@ while ($dist < 3)
          --let $casename=Autocommit update by pk
          update test.t1 set b=2 where a=0;
        }
+       if ($case == 3)
+       {
+         --let $casename=Unhinted table scan on new connection
+         connect(testcon, localhost, root,,);
+         --connection testcon
+           --eval set ndb_optimized_node_selection=$val;
+           begin;
+             select * from test.t1;
+             select * from test.t1 where a=0;
+           rollback;
+         --connection default
+         disconnect testcon;
+       }
        --dec $rpt
       }
       --enable_result_log
@@ -111,10 +125,10 @@ while ($dist < 3)
       # test.
       --echo $distname o-n-s $val case $casename
       eval select x.node_id,
-                  if (x.transcount >= $iter, "ALL",
-                    if (x.transcount <= 1, "NONE",
-                      if (x.transcount BETWEEN (($iter/2)-1)
-                                           AND (($iter/2)+1),
+                  if (x.transcount >= ($iter-($errbound)), "ALL",
+                    if (x.transcount <= ($errbound), "NONE",
+                      if (x.transcount BETWEEN (($iter/2)-($errbound))
+                                           AND (($iter/2)+($errbound)),
                           "HALF", "ERROR"))) as balance
            from (
              select a.node_id, b.transcount - a.transcount as transcount

storage/ndb/src/ndbapi/Ndbif.cpp

@@ -112,6 +112,27 @@ int Ndb::init(int aMaxNoOfTransactions) {
   theFirstTransId |=
       theImpl->m_ndb_cluster_connection.get_next_transid(theNdbBlockNumber);
 
+  /**
+   * Initialise current connect index and node iterator to get balanced
+   * node selection offset for first request using a new Ndb object
+   */
+  {
+    /**
+     * Use ndb_cluster_connection given id to provide RR variation
+     * of 'starting offset' for unhinted TC node choice.
+     */
+    const Uint32 tcNodeChoiceOffset =
+        (nodeId + theImpl->theCurrentConnectIndex) %
+        (theImpl->theNoOfDBnodes ? theImpl->theNoOfDBnodes : MAX_NDB_NODES);
+
+    /* Configure RR + proximity aware unhinted iterators */
+    theImpl->theCurrentConnectIndex = tcNodeChoiceOffset;
+    theImpl->m_ndb_cluster_connection.init_get_next_node(theImpl->m_node_iter);
+    for (Uint32 i = 0; i < tcNodeChoiceOffset; i++) {
+      theImpl->m_ndb_cluster_connection.get_next_node(theImpl->m_node_iter);
+    }
+  }
+
   /* Init cached min node version */
   theFacade->lock_poll_mutex();
   theCachedMinDbNodeVersion = theFacade->getMinDbNodeVersion();

storage/ndb/src/ndbapi/Ndbinit.cpp

@@ -118,7 +118,10 @@ void Ndb::setup(Ndb_cluster_connection *ndb_cluster_connection,
     }
   }
 
-  theImpl->m_ndb_cluster_connection.link_ndb_object(this);
+  const Uint64 id = theImpl->m_ndb_cluster_connection.link_ndb_object(this);
+
+  /* Give unhinted TC choice alg an offset to use in init() */
+  theImpl->theCurrentConnectIndex = id;
 
   DBUG_VOID_RETURN;
 }

storage/ndb/src/ndbapi/ndb_cluster_connection.cpp

@@ -495,6 +495,7 @@ Ndb_cluster_connection_impl::Ndb_cluster_connection_impl(
       m_latest_error_msg(),
       m_latest_error(0),
       m_data_node_neighbour(0),
+      m_num_created_ndb_objects(0),
       m_multi_wait_group(nullptr),
       m_uri_scheme(nullptr),
       m_uri_host(nullptr),
@@ -657,7 +658,7 @@ const Ndb *Ndb_cluster_connection::get_next_ndb_object(const Ndb *p) {
   return p->theImpl->m_next_ndb_object;
 }
 
-void Ndb_cluster_connection_impl::link_ndb_object(Ndb *p) {
+Uint64 Ndb_cluster_connection_impl::link_ndb_object(Ndb *p) {
   lock_ndb_objects();
   if (m_first_ndb_object != nullptr) {
     m_first_ndb_object->theImpl->m_prev_ndb_object = p;
@@ -666,10 +667,14 @@ void Ndb_cluster_connection_impl::link_ndb_object(Ndb *p) {
   p->theImpl->m_next_ndb_object = m_first_ndb_object;
   m_first_ndb_object = p;
 
+  const Uint64 id = m_num_created_ndb_objects++;
+
   // Wake up anyone waiting for changes to the Ndb instance list
   NdbCondition_Broadcast(m_new_delete_ndb_cond);
 
   unlock_ndb_objects();
+
+  return id;
 }
 
 void Ndb_cluster_connection_impl::unlink_ndb_object(Ndb *p) {

storage/ndb/src/ndbapi/ndb_cluster_connection_impl.hpp

@@ -174,7 +174,7 @@ class Ndb_cluster_connection_impl : public Ndb_cluster_connection {
   NdbMutex *m_new_delete_ndb_mutex;
   NdbCondition *m_new_delete_ndb_cond;
   Ndb *m_first_ndb_object;
-  void link_ndb_object(Ndb *);
+  Uint64 link_ndb_object(Ndb *);
   void unlink_ndb_object(Ndb *);
 
   BaseString m_latest_error_msg;
@@ -191,6 +191,9 @@ class Ndb_cluster_connection_impl : public Ndb_cluster_connection {
   // Closest data node neighbour
   Uint32 m_data_node_neighbour;
 
+  // Number of Ndb object creations
+  Uint64 m_num_created_ndb_objects;
+
   // Base offset for stats, from Ndb objects that are no
   // longer with us
   Uint64 globalApiStatsBaseline[Ndb::NumClientStatistics];