Bug #21178339: 'NDB_RESTORE -M' SOMETIMES: "FAILED TO CREATE INDEX ENTRY_KEY$UNIQUE" ERROR 1

Unique index creation requires a DBUTIL PREPARE which is initiated
from TRIX. DBUTIL allows only one PREPARE at a time. If a PREPARE
request is received in DBUTIL while a PREPARE is already running,
a PREPARE_SEIZE_ERROR is returned. This occurs if a CREATE_EVENT
has requested a PREPARE before unique index creation. For unique
index creation, the PREPARE_SEIZE_ERROR is not mapped to an
equivalent error code, so an error 1 "Unknown error" is returned.

Mapped PREPARE_SEIZE_ERROR to error 748 "Busy during read of event
table". Fixed ndb_restore to retry index creation on temporary
errors.

Author: Priyanka Sangam

mysql-test/suite/ndb/r/ndb_lock.result

@@ -198,3 +198,7 @@ lock tables t3 read;
 delete t2 from t2, t3 where t2.id = t3.id;
 unlock tables;
 drop table t2, t3;
+create table t1 (id int primary key, val int) engine=ndb;
+create unique index val_unique on t1(val);
+ERROR HY000: Got temporary error 748 'Busy during read of event table' from NDBCLUSTER
+drop table t1;

mysql-test/suite/ndb/t/ndb_lock.test

@@ -266,3 +266,10 @@ unlock tables;
 
 connection con1;
 drop table t2, t3;
+
+create table t1 (id int primary key, val int) engine=ndb;
+--exec $NDB_MGM --no-defaults --ndb-connectstring="$NDB_CONNECTSTRING" -e "all error 19000" >> $NDB_TOOLS_OUTPUT
+--error 1297
+create unique index val_unique on t1(val);
+--exec $NDB_MGM --no-defaults --ndb-connectstring="$NDB_CONNECTSTRING" -e "all error 0" >> $NDB_TOOLS_OUTPUT
+drop table t1;

storage/ndb/include/kernel/signaldata/BuildIndx.hpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -76,7 +76,8 @@ struct BuildIndxRef {
     AllocationFailure = 4252,
     InternalError = 4346,
     IndexNotFound = 4243,
-    DeadlockError = 4351
+    DeadlockError = 4351,
+    UtilBusy = 748
   };
 
   STATIC_CONST( SignalLength = 10 );

storage/ndb/src/kernel/blocks/ERROR_codes.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,13 +24,14 @@ Next DBDIH 7245
 Next DBTC 8109
 Next CMVMI 9000
 Next BACKUP 10042
-Next DBUTIL 11002
+Next PGMAN 11010
 Next DBTUX 12010
 Next SUMA 13049
 Next LGMAN 15001
 Next TSMAN 16001
 Next DBSPJ 17000
 Next TRIX 18000
+Next DBUTIL 19001
 
 TESTING NODE FAILURE, ARBITRATION
 ---------------------------------
@@ -795,3 +796,7 @@ Index stats:
 18022: fail RT_CLEAN_OLD (non-fatal) - as in 18021
 18023: fail RT_CLEAN_ALL (non-fatal) - as in 18021
 18024: fail RT_SCAN_FRAG - simulate TC error 630
+
+DBUTIL
+-------
+19000: fail UTIL_PREPARE_REQ with PREPARE_SEIZE_ERROR

storage/ndb/src/kernel/blocks/cmvmi/Cmvmi.cpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2003, 2015, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -1212,8 +1212,8 @@ void Cmvmi::execTAMPER_ORD(Signal* signal)
   }
   else if (errNo < 12000)
   {
-    // DBUTIL_REF ?
     jam();
+    tuserblockref = PGMAN_REF;
   }
   else if (errNo < 13000)
   {
@@ -1250,6 +1250,11 @@ void Cmvmi::execTAMPER_ORD(Signal* signal)
     jam();
     tuserblockref = TRIX_REF;
   }
+  else if (errNo < 20000)
+  {
+    jam();
+    tuserblockref = DBUTIL_REF;
+  }
   else if (errNo < 30000)
   {
     /*--------------------------------------------------------------------*/

storage/ndb/src/kernel/blocks/dbutil/DbUtil.cpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -42,6 +42,9 @@
 
 #include <NdbTick.h>
 
+#include <EventLogger.hpp>
+extern EventLogger * g_eventLogger;
+
 #include <signaldata/DbinfoScan.hpp>
 #include <signaldata/TransIdAI.hpp>
 
@@ -957,7 +960,6 @@ DbUtil::sendUtilPrepareRef(Signal* signal, UtilPrepareRef::ErrorCode error,
   ref->errorCode = error;
   ref->senderData = senderData;
   ref->dictErrCode = errCode2;
-
   sendSignal(recipient, GSN_UTIL_PREPARE_REF, signal, 
 	     UtilPrepareRef::SignalLength, JBB);
 }
@@ -1010,7 +1012,19 @@ DbUtil::execUTIL_PREPARE_REQ(Signal* signal)
   SectionHandle handle(this, signal);
 
   jam();
-  if (!c_runningPrepares.seizeFirst(prepPtr)) {
+
+  if(ERROR_INSERTED(19000))
+  {
+    jam();
+    CLEAR_ERROR_INSERT_VALUE;
+    g_eventLogger->info("Simulating DBUTIL prepare seize fail");
+    releaseSections(handle);
+    sendUtilPrepareRef(signal, UtilPrepareRef::PREPARE_SEIZE_ERROR,
+		       senderRef, senderData);
+    return;
+  }
+  if (!c_runningPrepares.seizeFirst(prepPtr))
+  {
     jam();
     releaseSections(handle);
     sendUtilPrepareRef(signal, UtilPrepareRef::PREPARE_SEIZE_ERROR,

storage/ndb/src/kernel/blocks/trix/Trix.cpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2003, 2013, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2003, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -701,6 +701,20 @@ void Trix::execUTIL_PREPARE_REF(Signal* signal)
   }
   subRecPtr.p = subRec;
   subRec->errorCode = (BuildIndxRef::ErrorCode)utilPrepareRef->errorCode;
+  switch (utilPrepareRef->errorCode) {
+  case UtilPrepareRef::PREPARE_SEIZE_ERROR:
+  case UtilPrepareRef::PREPARE_PAGES_SEIZE_ERROR:
+  case UtilPrepareRef::PREPARED_OPERATION_SEIZE_ERROR:
+  case UtilPrepareRef::DICT_TAB_INFO_ERROR:
+    subRec->errorCode = BuildIndxRef::UtilBusy;
+    break;
+  case UtilPrepareRef::MISSING_PROPERTIES_SECTION:
+    subRec->errorCode = BuildIndxRef::BadRequestType;
+    break;
+  default:
+    ndbrequire(false);
+    break;
+  }
 
   UtilReleaseConf* conf = (UtilReleaseConf*)signal->getDataPtrSend();
   conf->senderData = subRecPtr.i;

storage/ndb/tools/restore/consumer_restore.cpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2004, 2014, Oracle and/or its affiliates. All rights reserved.
+   Copyright (c) 2004, 2016, Oracle and/or its affiliates. All rights reserved.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
@@ -979,14 +979,36 @@ BackupRestore::rebuild_indexes(const TableS& table)
     const NDB_TICKS start = NdbTick_getCurrentTicks();
     info << "Rebuilding index `" << idx_name << "` on table `"
       << tab_name << "` ..." << flush;
-    if ((dict->getIndex(idx_name, tab_name) == NULL)
-        && (dict->createIndex(* idx, 1) != 0))
+    bool done = false;
+    for(int retries = 0; retries<11; retries++)
+    {
+      if ((dict->getIndex(idx_name, tab_name) == NULL)
+          && (dict->createIndex(* idx, 1) != 0))
+      {
+        if(dict->getNdbError().status == NdbError::TemporaryError)
+        {
+          err << "retry sleep 50 ms on error " <<
+                      dict->getNdbError().code << endl;
+          NdbSleep_MilliSleep(50);
+          continue;  // retry on temporary error
+        }
+        else
+        {
+          break;
+        }
+      }
+      else
+      {
+        done = true;
+        break;
+      }
+    }
+    if(!done)
     {
       info << "FAIL!" << endl;
       err << "Rebuilding index `" << idx_name << "` on table `"
         << tab_name <<"` failed: ";
       err << dict->getNdbError() << endl;
-
       return false;
     }
     const NDB_TICKS stop = NdbTick_getCurrentTicks();
@@ -2511,13 +2533,32 @@ BackupRestore::endOfTables(){
     idx->setName(split_idx[3].c_str());
     if (m_restore_meta && !m_disable_indexes && !m_rebuild_indexes)
     {
-      if (dict->createIndex(* idx) != 0)
+      bool done = false;
+      for(unsigned int retries = 0; retries < 11; retries++)
+      {
+        if(dict->createIndex(* idx) == 0)
+        {
+          done = true;  // success
+          break;
+        }
+        else if(dict->getNdbError().status == NdbError::TemporaryError)
+        {
+          err << "retry sleep 50 ms on error " <<
+                      dict->getNdbError().code << endl;
+          NdbSleep_MilliSleep(50);
+          continue;  // retry on temporary error
+        }
+        else
+        {
+          break; // error out on permanent error
+        }
+      }
+      if(!done)
       {
         delete idx;
         err << "Failed to create index `" << split_idx[3].c_str()
             << "` on `" << table_name << "`" << endl
             << dict->getNdbError() << endl;
-
         return false;
       }
       info << "Successfully created index `" << split_idx[3].c_str()