WL#12229 ATRT SUPPORTS TEST FAILURE RETRIES

Adds support for ATRT to re-run the test cases in case of failure.

Modifies ATRT to accept a configuration file where user can specifies the number
of retries if a test case fails. In case of failure test case, all the processes
will be restarted and the test will retried up to the no of times as specified in
config file. Setting a retry value in config file is optional, by default retry value
is '0' , where as the retry value less than zero will be treated as an error and retry
value greater than five, issues an warning.

Updated the storage/ndb/test/run-test/README general algorithm to include retry
algorithm.

changes made in storage/ndb/test/run-test/check-tests.sh to support configuration
if "max-retries" parameter is specified.

Author: Ragasudha Chillara

storage/ndb/test/run-test/README

@@ -1,3 +1,19 @@
+# Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301  USA
+
+
 run-test/README
 
 This document describes how atrt works and how to use it.
@@ -16,6 +32,7 @@ atrt has the following main loop:
   start each ndb_mgmd
   connect to each ndb_mgmd
   for each read(test case)
+  for (run = 1; run < 1 + test_max_retries; run++)
   do
     if previous test failed (or is first test)
       stop each ndbd

storage/ndb/test/run-test/atrt.hpp

@@ -121,6 +121,7 @@ struct atrt_testcase {
   time_t m_max_time;
   BaseString m_name;
   BaseString m_mysqld_options;
+  int m_max_retries;
 
   struct Command {
     atrt_process::Type m_cmd_type;

storage/ndb/test/run-test/check-tests.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-# Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2009, 2018, Oracle and/or its affiliates. All rights reserved.
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -59,6 +59,8 @@ check_file(){
 		testcase=$(expr $testcase + 4);;
 	    ^type:*)
 		;;
+	    ^max-retries:*)
+		;;
 	    ^$) 
                 if [ $testcase -ne 7 ]
 		then

storage/ndb/test/run-test/main.cpp

@@ -29,6 +29,7 @@
 #include <NdbSleep.h>
 
 #define PATH_SEPARATOR DIR_SEPARATOR
+#define TESTCASE_RETRIES_THRESHOLD_WARNING 5
 
 /** Global variables */
 static const char progname[] = "ndb_atrt";
@@ -325,161 +326,174 @@ int main(int argc, char **argv) {
    */
   g_logger.debug("Entering main loop");
   while (!feof(g_test_case_file)) {
-    /**
-     * Do we need to restart ndb
-     */
-    if (restart) {
-      restart = false;
-      g_logger.info("(Re)starting server processes...");
+    atrt_testcase test_case;
+    const int num_element_lines =
+        read_test_case(g_test_case_file, test_case, lineno);
+    if (num_element_lines == 0) {
+      // Should be at end of file.  Let while condition catch that.
+      continue;
+    }
+    if (num_element_lines < 0) {
+      g_logger.critical("Corrupt testcase at line %d (error %d)", lineno,
+                        num_element_lines);
+      goto cleanup;
+    }
+    g_logger.info("#%d - %s", test_no, test_case.m_name.c_str());
 
-      if (!stop_processes(g_config, ~0)) {
-        g_logger.critical("Failed to stop all processes");
-        goto end;
+    int result = 0;
+    time_t elapsed;
+    int testruns;
+    int total_runs = 1 + test_case.m_max_retries;
+    for (testruns = 1; testruns <= total_runs; testruns++) {
+      if (testruns > 1) {
+        g_logger.info("Retrying test #%d - '%s', attempt (%d/%d)", test_no,
+                      test_case.m_name.c_str(), testruns - 1,
+                      test_case.m_max_retries);
       }
 
-      g_logger.info("Waiting for all processes to stop...");
-      if (!wait_for_processes_to_stop(g_config, ~0)) {
-        g_logger.critical("Fail to stop all processes");
-        goto end;
-      }
+      /**
+      * Do we need to restart ndb
+      */
+      if (restart) {
+        restart = false;
+        g_logger.info("(Re)starting server processes...");
+
+        if (!stop_processes(g_config, ~0)) {
+          g_logger.critical("Failed to stop all processes");
+          goto end;
+        }
 
-      if (!setup_directories(g_config, 2)) {
-        g_logger.critical("Failed to setup directories");
-        goto end;
-      }
+        g_logger.info("Waiting for all processes to stop...");
+        if (!wait_for_processes_to_stop(g_config, ~0)) {
+          g_logger.critical("Fail to stop all processes");
+          goto end;
+        }
 
-      if (!setup_files(g_config, 2, 1)) {
-        g_logger.critical("Failed to setup files");
-        goto end;
-      }
+        if (!setup_directories(g_config, 2)) {
+          g_logger.critical("Failed to setup directories");
+          goto end;
+        }
 
-      if (!setup_hosts(g_config)) {
-        g_logger.critical("Failed to setup hosts");
-        goto end;
-      }
+        if (!setup_files(g_config, 2, 1)) {
+          g_logger.critical("Failed to setup files");
+          goto end;
+        }
 
-      g_logger.debug("Setup complete, starting servers");
-      if (!start(g_config, p_ndb | p_servers)) {
-        g_logger.critical("Failed to start server processes");
-        g_logger.info("Gathering logs and saving them as test %u", test_no);
+        if (!setup_hosts(g_config)) {
+          g_logger.critical("Failed to setup hosts");
+          goto end;
+        }
+
+        g_logger.debug("Setup complete, starting servers");
+        if (!start(g_config, p_ndb | p_servers)) {
+          g_logger.critical("Failed to start server processes");
+          g_logger.info("Gathering logs and saving them as test %u", test_no);
 
-        int tmp;
-        if (!gather_result(g_config, &tmp)) {
-          g_logger.critical("Failed to gather results");
+          int tmp;
+          if (!gather_result(g_config, &tmp)) {
+            g_logger.critical("Failed to gather results");
+            goto cleanup;
+          }
+
+          if (g_report_file != 0) {
+            fprintf(g_report_file, "%s ; %d ; %d ; %d ; %d\n", "start servers",
+                    test_no, ERR_FAILED_TO_START, 0, 0);
+            fflush(g_report_file);
+          }
+
+          BaseString resdir;
+          resdir.assfmt("result.%d", test_no);
+          remove_dir(resdir.c_str(), true);
+
+          if (rename("result", resdir.c_str()) != 0) {
+            g_logger.critical("Failed to rename %s as %s", "result",
+                              resdir.c_str());
+            goto cleanup;
+          }
           goto cleanup;
         }
 
-        if (g_report_file != 0) {
-          fprintf(g_report_file, "%s ; %d ; %d ; %d\n", "start servers",
-                  test_no, ERR_FAILED_TO_START, 0);
-          fflush(g_report_file);
+        if (!setup_db(g_config)) {
+          g_logger.critical("Failed to setup database");
+          goto cleanup;
         }
 
-        BaseString resdir;
-        resdir.assfmt("result.%d", test_no);
-        remove_dir(resdir.c_str(), true);
+        g_logger.info("All servers start completed");
+      }
 
-        if (rename("result", resdir.c_str()) != 0) {
-          g_logger.critical("Failed to rename %s as %s", "result",
-                            resdir.c_str());
-          goto cleanup;
-        }
+      // Assign processes to programs
+      if (!setup_test_case(g_config, test_case)) {
+        g_logger.critical("Failed to setup test case");
         goto cleanup;
       }
 
-      if (!setup_db(g_config)) {
-        g_logger.critical("Failed to setup database");
+      if (!start_processes(g_config, p_clients)) {
+        g_logger.critical("Failed to start client processes");
         goto cleanup;
       }
 
-      g_logger.info("All servers start completed");
-    }
+      const time_t start = time(0);
+      time_t now = start;
+      do {
+        if (!update_status(g_config, atrt_process::AP_ALL)) {
+          g_logger.critical("Failed to get updated status for all processes");
+          goto cleanup;
+        }
 
-    // const int start_line = lineno;
-    atrt_testcase test_case;
-    const int num_element_lines =
-        read_test_case(g_test_case_file, test_case, lineno);
-    if (num_element_lines == 0) {
-      // Should be at end of file.  Let while condition catch that.
-      continue;
-    }
-    if (num_element_lines < 0) {
-      g_logger.critical("Corrupt testcase at line %d (error %d)", lineno,
-                        num_element_lines);
-      goto cleanup;
-    }
-    g_logger.info("#%d - %s", test_no, test_case.m_name.c_str());
+        if ((result = check_ndb_or_servers_failures(g_config))) {
+          break;
+        }
 
-    // Assign processes to programs
-    if (!setup_test_case(g_config, test_case)) {
-      g_logger.critical("Failed to setup test case");
-      goto cleanup;
-    }
+        if (!is_client_running(g_config)) {
+          break;
+        }
 
-    if (!start_processes(g_config, p_clients)) {
-      g_logger.critical("Failed to start client processes");
-      goto cleanup;
-    }
+        if (!do_command(g_config)) {
+          result = ERR_COMMAND_FAILED;
+          g_logger.critical("Failure on client command execution");
+          break;
+        }
 
-    int result = 0;
+        now = time(0);
+        if (now > (start + test_case.m_max_time)) {
+          g_logger.debug("Timed out");
+          result = ERR_MAX_TIME_ELAPSED;
+          g_logger.info("Timeout '%s' after %ld seconds",
+                        test_case.m_name.c_str(), test_case.m_max_time);
+          break;
+        }
+        NdbSleep_SecSleep(1);
+      } while (true);
 
-    const time_t start = time(0);
-    time_t now = start;
-    do {
-      if (!update_status(g_config, atrt_process::AP_ALL)) {
-        g_logger.critical("Failed to get updated status for all processes");
+      elapsed = time(0) - start;
+      if (!stop_processes(g_config, p_clients)) {
+        g_logger.critical("Failed to stop client processes");
         goto cleanup;
       }
 
-      if ((result = check_ndb_or_servers_failures(g_config))) {
-        break;
-      }
-
-      if (!is_client_running(g_config)) {
-        break;
+      if (!wait_for_processes_to_stop(g_config, p_clients)) {
+        g_logger.critical("Failed to stop client processes");
+        goto cleanup;
       }
 
-      if (!do_command(g_config)) {
-        result = ERR_COMMAND_FAILED;
-        g_logger.critical("Failure on client command execution");
-        break;
+      int tmp, *rp = result ? &tmp : &result;
+      if (!gather_result(g_config, rp)) {
+        g_logger.critical("Failed to gather result after test run");
+        goto end;
       }
 
-      now = time(0);
-      if (now > (start + test_case.m_max_time)) {
-        g_logger.debug("Timed out");
-        result = ERR_MAX_TIME_ELAPSED;
-        g_logger.info("Timeout '%s' after %ld seconds",
-                      test_case.m_name.c_str(), test_case.m_max_time);
+      g_logger.info("#%d %s(%d)", test_no, (result == 0 ? "OK" : "FAILED"),
+                    result);
+      if (result == 0) {
         break;
+      } else {
+        restart = true;
       }
-      NdbSleep_SecSleep(1);
-    } while (true);
-
-    const time_t elapsed = time(0) - start;
-
-    if (!stop_processes(g_config, p_clients)) {
-      g_logger.critical("Failed to stop client processes");
-      goto cleanup;
-    }
-
-    if (!wait_for_processes_to_stop(g_config, p_clients)) {
-      g_logger.critical("Failed to stop client processes");
-      goto cleanup;
     }
 
-    int tmp, *rp = result ? &tmp : &result;
-    if (!gather_result(g_config, rp)) {
-      g_logger.critical("Failed to gather result after test run");
-      goto end;
-    }
-
-    g_logger.info("#%d %s(%d)", test_no, (result == 0 ? "OK" : "FAILED"),
-                  result);
-
     if (g_report_file != 0) {
-      fprintf(g_report_file, "%s ; %d ; %d ; %ld\n", test_case.m_name.c_str(),
-              test_no, result, elapsed);
+      fprintf(g_report_file, "%s ; %d ; %d ; %ld ; %d\n",
+              test_case.m_name.c_str(), test_no, result, elapsed, testruns);
       fflush(g_report_file);
     }
 
@@ -510,10 +524,6 @@ int main(int argc, char **argv) {
     if (reset_config(g_config)) {
       restart = true;
     }
-
-    if (result != 0) {
-      restart = true;
-    }
     test_no++;
   }
   return_code = 0;
@@ -526,8 +536,8 @@ int main(int argc, char **argv) {
 end:
   g_logger.info("Finishing, result: %d", return_code);
   if (return_code != 0 && g_report_file != 0) {
-    fprintf(g_report_file, "%s ; %d ; %d ; %d\n", "critical error", test_no,
-            ERR_FAILED_TO_START, 0);
+    fprintf(g_report_file, "%s ; %d ; %d ; %d ; %d\n", "critical error",
+            test_no, ERR_FAILED_TO_START, 0, 0);
     fflush(g_report_file);
   }
   if (g_report_file != 0) {
@@ -1379,6 +1389,23 @@ int read_test_case(FILE *file, atrt_testcase &tc, int &line) {
     used_elements++;
   }
 
+  tc.m_max_retries = 0;
+  if (p.get("max-retries", &mt)) {
+    tc.m_max_retries = atoi(mt);
+    used_elements++;
+  }
+
+  if (tc.m_max_retries < 0) {
+    g_logger.error("No of retries must not be less than zero for test '%s'",
+                   tc.m_name.c_str());
+    return -4;
+  }
+
+  if (tc.m_max_retries > TESTCASE_RETRIES_THRESHOLD_WARNING)
+    g_logger.warning(
+        "No of retries should be less than or equal to %d for test '%s'",
+        TESTCASE_RETRIES_THRESHOLD_WARNING, tc.m_name.c_str());
+
   if (used_elements != elements) {
     g_logger.critical(
         "Invalid test file: unknown properties in test case above line: %d",