Fix the clean up of staging directory for function test suite

connector/build.sbt

@@ -32,7 +32,7 @@ libraryDependencies += "org.scala-lang.modules" %% "scala-parser-combinators" %
 libraryDependencies += "com.vertica.jdbc" % "vertica-jdbc" % "24.4.0-0"
 libraryDependencies += "org.apache.spark" %% "spark-core" % "3.5.5"
 libraryDependencies += "org.apache.spark" %% "spark-sql" % "3.5.5"
-libraryDependencies += "org.apache.hadoop" % "hadoop-hdfs" % "3.3.2"
+libraryDependencies += "org.apache.hadoop" % "hadoop-hdfs" % "3.3.4"
 libraryDependencies += "org.scalactic" %% "scalactic" % "3.2.16" % Test
 libraryDependencies += "org.scalatest" %% "scalatest" % "3.2.16" % "test"
 libraryDependencies += "com.typesafe.scala-logging" %% "scala-logging" % "3.9.5"

functional-tests/build.sbt

@@ -33,7 +33,7 @@ val sparkVersion = Option(System.getProperty("sparkVersion")) match {
 
 val hadoopVersion = Option(System.getProperty("hadoopVersion")) match {
   case Some(hadoopVersion) => hadoopVersion
-  case None => sys.env.getOrElse("HADOOP_VERSION", "3.3.2")
+  case None => sys.env.getOrElse("HADOOP_VERSION", "3.3.4")
 }
 
 resolvers += "Artima Maven Repository" at "https://repo.artima.com/releases"

functional-tests/src/main/scala/com/vertica/spark/functests/endtoend/EndToEndTests.scala

@@ -65,10 +65,33 @@ abstract class EndToEnd(readOpts: Map[String, String], writeOpts: Map[String, St
     .getOrCreate()
 
   override def afterEach(): Unit = {
+    try {
+      val anyFiles= fsLayer.getFileList(fsConfig.address)
+      anyFiles match {
+        case Right(files) =>
+          if (files.nonEmpty) {
+            files.foreach { file =>
+              try {
+                fsLayer.removeFile(file)
+              } catch {
+                case e: Exception =>
+                  // Handle the exception here, for example:
+                  println(s"An exception occurred while deleting file: ${file}, ${e.getMessage}")
+                  e.printStackTrace()
+              }
+            }
+          }
+      }
+    } catch {
+      case e: Exception =>
+        // Handle the exception here, for example:
+        println(s"An exception occurred while removing or creating the directory: ${e.getMessage}")
+        e.printStackTrace()
+    }
     val anyFiles= fsLayer.getFileList(fsConfig.address)
     anyFiles match {
       case Right(files) =>
-        if(files.nonEmpty) assert(files.isEmpty, ". After each test, staging directory should be cleaned.")
+        if(files.nonEmpty) assert(files.isEmpty, ". After each test, staging directory should be cleaned. fsConfig.address" + fsConfig.address)
       case Left(_) => fail("Error getting file list from " + fsConfig.address)
     }
   }