[FLINK-26105][e2e] Fixes log file extension

Rolling log file naming strategy is enabled by default which
might cause test instabilities do to the relevant substring
not being present in the *.log file. This change expands the
grep and find calls to also consider *.log.[0-9]+ files.

Author: XComp

flink-end-to-end-tests/test-scripts/common.sh

@@ -488,7 +488,7 @@ function wait_for_job_state_transition {
   echo "Waiting for job ($job) to switch from state ${initial_state} to state ${next_state} ..."
 
   while : ; do
-    N=$(grep -o "($job) switched from state ${initial_state} to ${next_state}" $FLINK_LOG_DIR/*standalonesession*.log | tail -1)
+    N=$(grep -o "($job) switched from state ${initial_state} to ${next_state}" $FLINK_LOG_DIR/*standalonesession*.log* | tail -1)
 
     if [[ -z $N ]]; then
       sleep 1
@@ -523,7 +523,7 @@ function wait_job_terminal_state {
   echo "Waiting for job ($job) to reach terminal state $expected_terminal_state ..."
 
   while : ; do
-    local N=$(grep -o "Job $job reached terminal state .*" $FLINK_LOG_DIR/*$log_file_name*.log | tail -1 || true)
+    local N=$(grep -o "Job $job reached terminal state .*" $FLINK_LOG_DIR/*$log_file_name*.log* | tail -1 || true)
     if [[ -z $N ]]; then
       sleep 1
     else
@@ -645,7 +645,7 @@ function get_job_metric {
 function get_metric_processed_records {
   OPERATOR=$1
   JOB_NAME="${2:-General purpose test job}"
-  N=$(grep ".${JOB_NAME}.$OPERATOR.numRecordsIn:" $FLINK_LOG_DIR/*taskexecutor*.log | sed 's/.* //g' | tail -1)
+  N=$(grep ".${JOB_NAME}.$OPERATOR.numRecordsIn:" $FLINK_LOG_DIR/*taskexecutor*.log* | sed 's/.* //g' | tail -1)
   if [ -z $N ]; then
     N=0
   fi
@@ -655,7 +655,7 @@ function get_metric_processed_records {
 function get_num_metric_samples {
   OPERATOR=$1
   JOB_NAME="${2:-General purpose test job}"
-  N=$(grep ".${JOB_NAME}.$OPERATOR.numRecordsIn:" $FLINK_LOG_DIR/*taskexecutor*.log | wc -l)
+  N=$(grep ".${JOB_NAME}.$OPERATOR.numRecordsIn:" $FLINK_LOG_DIR/*taskexecutor*.log* | wc -l)
   if [ -z $N ]; then
     N=0
   fi
@@ -705,7 +705,7 @@ function wait_num_of_occurence_in_logs {
     echo "Waiting for text ${text} to appear ${number} of times in logs..."
 
     while : ; do
-      N=$(grep -o "${text}" $FLINK_LOG_DIR/*${logs}*.log | wc -l)
+      N=$(grep -o "${text}" $FLINK_LOG_DIR/*${logs}*.log* | wc -l)
 
       if [ -z $N ]; then
         N=0
@@ -734,7 +734,7 @@ function wait_num_checkpoints {
     echo "Waiting for job ($JOB) to have at least $NUM_CHECKPOINTS completed checkpoints ..."
 
     while : ; do
-      N=$(grep -o "Completed checkpoint [1-9]* for job $JOB" $FLINK_LOG_DIR/*standalonesession*.log | awk '{print $3}' | tail -1)
+      N=$(grep -o "Completed checkpoint [1-9]* for job $JOB" $FLINK_LOG_DIR/*standalonesession*.log* | awk '{print $3}' | tail -1)
 
       if [ -z $N ]; then
         N=0
@@ -776,7 +776,7 @@ function expect_in_taskmanager_logs {
     local expected="$1"
     local timeout=$2
     local i=0
-    local logfile="$FLINK_LOG_DIR/flink*taskexecutor*log"
+    local logfile="$FLINK_LOG_DIR/flink*taskexecutor*log*"
 
 
     while ! grep "${expected}" ${logfile} > /dev/null; do

flink-end-to-end-tests/test-scripts/common_ha.sh

@@ -49,7 +49,7 @@ function verify_num_occurences_in_logs() {
     local text="$2"
     local expected_no="$3"
 
-    local actual_no=$(grep -r --include "*${log_pattern}*.log" -e "${text}" "$FLINK_LOG_DIR/" | cut -d ":" -f 1 | uniq | wc -l)
+    local actual_no=$(grep -r --include "*${log_pattern}*.log*" -e "${text}" "$FLINK_LOG_DIR/" | cut -d ":" -f 1 | sed "s/\.[0-9]\{1,\}$//g" | uniq | wc -l)
     [[ "${expected_no}" -eq "${actual_no}" ]]
 }
 

flink-end-to-end-tests/test-scripts/queryable_state_base.sh

@@ -25,7 +25,7 @@ function link_queryable_state_lib {
 
 # Returns the ip address of the queryable state server
 function get_queryable_state_server_ip {
-    local ip=$(cat $FLINK_LOG_DIR/flink*taskexecutor*log \
+    local ip=$(cat $FLINK_LOG_DIR/flink*taskexecutor*log* \
         | grep "Started Queryable State Server" \
         | head -1 \
         | grep -Eo  "\.*[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.*")
@@ -35,7 +35,7 @@ function get_queryable_state_server_ip {
 
 # Returns the ip address of the queryable state server
 function get_queryable_state_proxy_port {
-    local port=$(cat $FLINK_LOG_DIR/flink*taskexecutor*log \
+    local port=$(cat $FLINK_LOG_DIR/flink*taskexecutor*log* \
         | grep "Started Queryable State Proxy Server" \
         | head -1 \
         | grep -Eo  "\.*\:([0-9]{3,5})\.*" | tr -d ":.")

flink-end-to-end-tests/test-scripts/test_cli.sh

@@ -65,7 +65,7 @@ function extract_valid_job_list_by_type_from_job_list_return() {
 }
 
 function extract_task_manager_slot_request_count() {
-    COUNT=`grep "Receive slot request" $FLINK_LOG_DIR/*taskexecutor*.log | wc -l`
+    COUNT=`grep "Receive slot request" $FLINK_LOG_DIR/*taskexecutor*.log* | wc -l`
     echo $COUNT
 }
 

flink-end-to-end-tests/test-scripts/test_rocksdb_state_memory_control.sh

@@ -80,7 +80,7 @@ function buildBaseJobCmd {
 function find_max_block_cache_usage() {
   OPERATOR=$1
   JOB_NAME="${2:-General purpose test job}"
-  N=$(grep ".${JOB_NAME}.$OPERATOR.rocksdb.block-cache-usage:" $FLINK_LOG_DIR/*taskexecutor*.log | sed 's/.* //g' | sort -rn | head -n 1)
+  N=$(grep ".${JOB_NAME}.$OPERATOR.rocksdb.block-cache-usage:" $FLINK_LOG_DIR/*taskexecutor*.log* | sed 's/.* //g' | sort -rn | head -n 1)
   if [ -z $N ]; then
     N=0
   fi

tools/ci/controller_utils.sh

@@ -33,7 +33,7 @@ print_system_info() {
 
 # locate YARN logs and put them into artifacts directory
 put_yarn_logs_to_artifacts() {
-	for file in `find ./flink-yarn-tests/target -type f -name '*.log' -or -name '*.out'`; do
+	for file in `find ./flink-yarn-tests/target -type f -name '*.log*' -or -name '*.out'`; do
 		TARGET_FILE=`echo "$file" | grep -Eo "container_[0-9_]+/(.*).[a-z]{3}"`
 		TARGET_DIR=`dirname	 "$TARGET_FILE"`
 		mkdir -p "$DEBUG_FILES_OUTPUT_DIR/yarn-tests/$TARGET_DIR"

tools/ci/maven-utils.sh

@@ -74,7 +74,7 @@ function collect_coredumps {
 	local SEARCHDIR=$1
 	local TARGET_DIR=$2
 	echo "Searching for .dump, .dumpstream and related files in '$SEARCHDIR'"
-	for file in `find $SEARCHDIR -type f -regextype posix-extended -iregex '.*\.hprof|.*\.dump|.*\.dumpstream|.*hs.*\.log|.*/core(.[0-9]+)?$'`; do
+	for file in `find $SEARCHDIR -type f -regextype posix-extended -iregex '.*\.hprof|.*\.dump|.*\.dumpstream|.*hs.*\.log(\.[0-9]+)?|.*/core(\.[0-9]+)?$'`; do
 		echo "Moving '$file' to target directory ('$TARGET_DIR')"
 		mv $file $TARGET_DIR/$(echo $file | tr "/" "-")
 	done