Bug#34013385 TUP require failure

The combination of
 - Batching with multiple in-flight operations per key
 - IgnoreError execution mode
 - Transient errors occurring on non-primary replicas

can lead to TUP level inconsistencies which result in replica
misalignment and other require assertion failures.

This combination and its side-effects are made impossible by detecting
when operations are failing on non-primary replicas, and forcing
AbortOnError handling (e.g. rollback) for the containing transaction.

An assertion checking for this condition in ACC is upgraded to a
require.

This behavioural change is verified with a new testcase :
  testBasic -n AbortIgnoreError

This test is added to daily-devel tests.

In addition, the test framework UtilTransactions class has a number
of replica and index consistency check mechanisms added, which are also
made available via the enhanced verify_index tool, allowing replica and
index consistency to be verified from any programmatic or scripted
testcase.

Approved by : Maitrayi Sabaratnam <[email protected]>

Author: frazerclement

storage/ndb/include/kernel/signaldata/LqhKey.hpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2003, 2021, Oracle and/or its affiliates.
+   Copyright (c) 2003, 2022, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -853,7 +853,8 @@ class LqhKeyRef {
   friend bool printLQHKEYREF(FILE * output, const Uint32 * theData, Uint32 len, Uint16 receiverBlockNo);
 
 public:
-  STATIC_CONST( SignalLength = 5 );
+  STATIC_CONST( SignalLengthWithoutFlags = 5 );
+  STATIC_CONST( SignalLength = 6 );
 
 private:
 
@@ -865,8 +866,30 @@ class LqhKeyRef {
   Uint32 errorCode;
   Uint32 transId1;
   Uint32 transId2;
+  Uint32 flags;
+
+  static Uint32 getReplicaErrorFlag(const Uint32& flags);
+  static void setReplicaErrorFlag(Uint32& flags, Uint32 val);
+
+  enum Flags {
+    LKR_REPLICA_ERROR_SHIFT  = 0
+  };
 };
 
+inline
+Uint32
+LqhKeyRef::getReplicaErrorFlag(const Uint32& flags)
+{
+  return ((flags >> LKR_REPLICA_ERROR_SHIFT) & 0x1);
+}
+
+inline
+void
+LqhKeyRef::setReplicaErrorFlag(Uint32& flags, Uint32 val)
+{
+  ASSERT_BOOL(val, "LqhKeyRef::setReplicaErrorFlag");
+  flags |= (val << LKR_REPLICA_ERROR_SHIFT);
+}
 
 #undef JAM_FILE_ID
 

storage/ndb/src/kernel/blocks/ERROR_codes.txt

@@ -1,4 +1,4 @@
-# Copyright (c) 2003, 2021, Oracle and/or its affiliates.
+# Copyright (c) 2003, 2022, Oracle and/or its affiliates.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License, version 2.0,
@@ -25,7 +25,7 @@ Next NDBCNTR 1023
 Next NDBFS 2003
 Next DBACC 3005
 Next DBTUP 4039
-Next DBLQH 5098
+Next DBLQH 5109
 Next DBDICT 6223
 Next DBDIH 7249
 Next DBTC 8115
@@ -352,6 +352,9 @@ Error insertions used to test ACCKEYCONF/ABORT interleaving
       ACCKEYCONF signal clears ERROR Insertion
 5097: ndbrequire(false)
 
+5107: Cause SimulatedBlock::allocChunks() to return chunks with gaps
+5108: Cause request to fail with error 1218 SendBuffer overload (temp error)
+ 
 
 
 ERROR CODES FOR TESTING TIME-OUT HANDLING IN DBTC

storage/ndb/src/kernel/blocks/dbacc/DbaccMain.cpp

@@ -4987,7 +4987,7 @@ void Dbacc::commitOperation(Signal* signal)
   Uint32 opbits = operationRecPtr.p->m_op_bits;
   Uint32 op = opbits & Operationrec::OP_MASK;
   ndbrequire((opbits & Operationrec::OP_STATE_MASK) == Operationrec::OP_STATE_EXECUTED);
-  ndbassert((opbits & Operationrec::OP_PENDING_ABORT) == 0);
+  ndbrequire((opbits & Operationrec::OP_PENDING_ABORT) == 0);
   if ((opbits & Operationrec::OP_COMMIT_DELETE_CHECK) == 0 && 
       (op != ZREAD && op != ZSCAN_OP))
   {

storage/ndb/src/kernel/blocks/dblqh/Dblqh.hpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2003, 2021, Oracle and/or its affiliates.
+   Copyright (c) 2003, 2022, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -2193,7 +2193,8 @@ class Dblqh
       NEW_FROM_TC = 2,
       REQ_FROM_TC = 3,
       ABORT_FROM_TC = 4,
-      ABORT_FROM_LQH = 5
+      ABORT_FROM_LQH = 5,
+      ABORT_FROM_LQH_REPLICA = 6
     };
     enum TransactionState {
       IDLE = 0,

storage/ndb/src/kernel/blocks/dblqh/DblqhMain.cpp

@@ -3320,6 +3320,7 @@ void Dblqh::earlyKeyReqAbort(Signal* signal,
     ref->errorCode = errCode;
     ref->transId1 = transid1;
     ref->transId2 = transid2;
+    ref->flags = 0;
     sendSignal(signal->senderBlockRef(), GSN_LQHKEYREF, signal, 
 	       LqhKeyRef::SignalLength, JBB);
   }//if
@@ -3497,7 +3498,10 @@ void Dblqh::execLQHKEYREF(Signal* signal)
       warningReport(signal, 15);
       return;
     }//if
-    abortErrorLab(signal);
+    /* Mark abort due to replica issue */
+    regTcPtr->abortState = TcConnectionrec::ABORT_FROM_LQH_REPLICA;
+    regTcPtr->errorCode = terrorCode;
+    abortCommonLab(signal);
     return;
     break;
   case TcConnectionrec::LOG_CONNECTED:
@@ -4965,6 +4969,7 @@ void Dblqh::execLQHKEYREQ(Signal* signal)
   }
 
   if (ERROR_INSERTED_CLEAR(5047) ||
+      ERROR_INSERTED_CLEAR(5108) ||
       ERROR_INSERTED(5079) ||
      (ERROR_INSERTED(5102) &&
       LqhKeyReq::getNoTriggersFlag(Treqinfo)) ||
@@ -10262,7 +10267,9 @@ void Dblqh::continueAfterLogAbortWriteLab(Signal* signal)
     cleanUp(signal);
     return;
   }//if
-  if (regTcPtr->abortState == TcConnectionrec::ABORT_FROM_LQH) {
+  if ((regTcPtr->abortState == TcConnectionrec::ABORT_FROM_LQH) ||
+      (regTcPtr->abortState == TcConnectionrec::ABORT_FROM_LQH_REPLICA))
+  {
     LqhKeyRef * const lqhKeyRef = (LqhKeyRef *)signal->getDataPtrSend();
 
     jam();
@@ -10271,6 +10278,12 @@ void Dblqh::continueAfterLogAbortWriteLab(Signal* signal)
     lqhKeyRef->errorCode = regTcPtr->errorCode;
     lqhKeyRef->transId1 = regTcPtr->transid[0];
     lqhKeyRef->transId2 = regTcPtr->transid[1];
+    lqhKeyRef->flags = 0;
+    if (regTcPtr->abortState == TcConnectionrec::ABORT_FROM_LQH_REPLICA)
+    {
+      jam();
+      LqhKeyRef::setReplicaErrorFlag(lqhKeyRef->flags, 1);
+    }
     sendSignal(regTcPtr->clientBlockref, GSN_LQHKEYREF, signal, 
                LqhKeyRef::SignalLength, JBB);
   } else if (regTcPtr->abortState == TcConnectionrec::ABORT_FROM_TC) {

storage/ndb/src/kernel/blocks/dbspj/DbspjMain.cpp

@@ -1,5 +1,5 @@
 /*
-   Copyright (c) 2012, 2021, Oracle and/or its affiliates.
+   Copyright (c) 2012, 2022, Oracle and/or its affiliates.
 
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License, version 2.0,
@@ -1108,6 +1108,7 @@ Dbspj::handle_early_lqhkey_ref(Signal* signal,
     ref->errorCode = err;
     ref->transId1 = transid[0];
     ref->transId2 = transid[1];
+    ref->flags = 0;
     sendSignal(returnref, GSN_LQHKEYREF, signal,
                LqhKeyRef::SignalLength, JBB);
   }

storage/ndb/src/kernel/blocks/dbtc/DbtcMain.cpp

@@ -7489,15 +7489,26 @@ void Dbtc::execLQHKEYREF(Signal* signal)
 	warningReport(signal, 25);
 	return;
       }//if
-
+      Uint32 flags = 0;
+      if (signal->getLength() >= LqhKeyRef::SignalLength)
+      {
+        jam();
+        flags = lqhKeyRef->flags;
+      }
+      /**
+       * If the error came from a backup replica rather than the primary
+       * then we will abort the transaction in all cases.
+       */
+      const bool needAbort = (LqhKeyRef::getReplicaErrorFlag(flags) != 0);
       const Uint32 triggeringOp = regTcPtr->triggeringOperation;
       ConnectionState TapiConnectstate = regApiPtr->apiConnectstate;
 
       time_track_complete_key_operation_error(regTcPtr,
                                     refToNode(regApiPtr->ndbapiBlockref),
                                     regTcPtr->tcNodedata[0]);
 
-      if (unlikely(TapiConnectstate == CS_ABORTING))
+      if (unlikely(TapiConnectstate == CS_ABORTING ||
+                   needAbort))
       {
         jam();
         goto do_abort;
@@ -7667,7 +7678,7 @@ void Dbtc::execLQHKEYREF(Signal* signal)
       }
 
       const Uint32 abort = regTcPtr->m_execAbortOption;
-      if (abort == TcKeyReq::AbortOnError || triggeringOp != RNIL) {
+      if (abort == TcKeyReq::AbortOnError || triggeringOp != RNIL || needAbort) {
 	/**
 	 * No error is allowed on this operation
 	 */