Make crt dependency optional in transfer manager module (#3613)

* Make aws-crt an optional dependency in s3-transfer-manager module.

* Update README

Author: zoewangg

.changes/next-release/removal-S3TransferManager-2c2c795.json

@@ -0,0 +1,6 @@
+{
+    "type": "removal",
+    "category": "S3 Transfer Manager",
+    "contributor": "",
+    "description": "Make `aws-crt` an optional dependency in `s3-transfer-manager` module. Customers need to explicitly add `aws-crt` dependency if they want to use CRT-based Transfer Manager"
+}

services-custom/s3-transfer-manager/README.md

@@ -9,18 +9,23 @@ as well as pause the transfer for execution at a later time.
 
 ### Add a dependency for the S3 Transfer Manager 
 
-First, you need to include the dependency in your project.
+First, you need to include `s3-transfer-manager` and `aws-crt` in your project.
 
 ```xml
 <dependency>
   <groupId>software.amazon.awssdk</groupId>
   <artifactId>s3-transfer-manager</artifactId>
   <version>${awsjavasdk.version}-PREVIEW</version>
 </dependency>
+<dependency>
+  <groupId>software.amazon.awssdk.crt</groupId>
+  <artifactId>aws-crt</artifactId>
+  <version>${awscrt.version}</version>
+</dependency>
 ```
 
-Note that you need to replace `${awsjavasdk.version}` with the latest
-SDK version.
+Note that you need to replace `${awsjavasdk.version}` and `${awscrt.version}` with the latest
+version.
 
 ### Instantiate the S3 Transfer Manager
 

services-custom/s3-transfer-manager/pom.xml

@@ -80,6 +80,8 @@
             <groupId>software.amazon.awssdk.crt</groupId>
             <artifactId>aws-crt</artifactId>
             <version>${awscrt.version}</version>
+            <!-- Only required for CRT-based TM -->
+            <optional>true</optional>
         </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/S3TransferManager.java

@@ -678,6 +678,11 @@ default Copy copy(Consumer<CopyRequest.Builder> copyRequestBuilder) {
 
     /**
      * Create an {@code S3TransferManager} using the default values.
+     * <p>
+     * The type of {@link S3AsyncClient} used depends on if AWS Common Runtime (CRT) library
+     * {@code software.amazon.awssdk.crt:crt} is on the classpath. If CRT is available, a CRT-based S3 client will be created
+     * ({@link S3AsyncClient#crtCreate()}). Otherwise, a standard S3 client({@link S3AsyncClient#create()}) will be created. Note
+     * that only CRT-based S3 client supports parallel transfer for now, so it's recommended to add CRT as a dependency.
      */
     static S3TransferManager create() {
         return builder().build();

services-custom/s3-transfer-manager/src/main/java/software/amazon/awssdk/transfer/s3/internal/DefaultS3TransferManager.java

@@ -26,6 +26,7 @@
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.Executor;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 import software.amazon.awssdk.annotations.SdkInternalApi;
 import software.amazon.awssdk.annotations.SdkTestInternalApi;
 import software.amazon.awssdk.arns.Arn;
@@ -36,6 +37,7 @@
 import software.amazon.awssdk.core.exception.SdkClientException;
 import software.amazon.awssdk.core.exception.SdkException;
 import software.amazon.awssdk.core.internal.async.FileAsyncRequestBody;
+import software.amazon.awssdk.core.internal.util.ClassLoaderHelper;
 import software.amazon.awssdk.crt.s3.ResumeToken;
 import software.amazon.awssdk.http.SdkHttpExecutionAttributes;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
@@ -102,8 +104,14 @@ public final class DefaultS3TransferManager implements S3TransferManager {
 
     public DefaultS3TransferManager(DefaultBuilder tmBuilder) {
         transferConfiguration = resolveTransferManagerConfiguration(tmBuilder);
-        isDefaultS3AsyncClient = tmBuilder.s3AsyncClient == null;
-        s3AsyncClient = isDefaultS3AsyncClient ? S3AsyncClient.crtCreate() : tmBuilder.s3AsyncClient;
+        if (tmBuilder.s3AsyncClient == null) {
+            isDefaultS3AsyncClient = true;
+            s3AsyncClient = defaultS3AsyncClient().get();
+        } else {
+            isDefaultS3AsyncClient = false;
+            s3AsyncClient = tmBuilder.s3AsyncClient;
+        }
+
         uploadDirectoryHelper = new UploadDirectoryHelper(transferConfiguration, this::uploadFile);
         ListObjectsHelper listObjectsHelper = new ListObjectsHelper(s3AsyncClient::listObjectsV2);
         downloadDirectoryHelper = new DownloadDirectoryHelper(transferConfiguration,
@@ -134,6 +142,22 @@ public DefaultS3TransferManager(DefaultBuilder tmBuilder) {
         s3ClientType = s3CrtAsyncClient instanceof S3CrtAsyncClient ? S3ClientType.CRT_BASED : S3ClientType.JAVA_BASED;
     }
 
+    private static Supplier<S3AsyncClient> defaultS3AsyncClient() {
+        if (crtInClasspath()) {
+            return S3AsyncClient::crtCreate;
+        }
+        return S3AsyncClient::create;
+    }
+
+    private static boolean crtInClasspath() {
+        try {
+            ClassLoaderHelper.loadClass("software.amazon.awssdk.crt.s3.S3Client", false);
+        } catch (ClassNotFoundException e) {
+            return false;
+        }
+        return true;
+    }
+
     private static TransferManagerConfiguration resolveTransferManagerConfiguration(DefaultBuilder tmBuilder) {
         TransferManagerConfiguration.Builder transferConfigBuilder = TransferManagerConfiguration.builder();
         transferConfigBuilder.uploadDirectoryFollowSymbolicLinks(tmBuilder.uploadDirectoryFollowSymbolicLinks);

services-custom/s3-transfer-manager/src/test/java/software/amazon/awssdk/transfer/s3/internal/S3TransferManagerTest.java

@@ -58,9 +58,6 @@ class S3TransferManagerTest {
     private DownloadDirectoryHelper downloadDirectoryHelper;
     private TransferManagerConfiguration configuration;
 
-
-    private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
-
     @BeforeEach
     public void methodSetup() {
         mockS3Crt = mock(S3CrtAsyncClient.class);

test/s3-benchmarks/pom.xml

@@ -56,6 +56,11 @@
             <artifactId>s3-transfer-manager</artifactId>
             <version>${awsjavasdk.version}-PREVIEW</version>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk.crt</groupId>
+            <artifactId>aws-crt</artifactId>
+            <version>${awscrt.version}</version>
+        </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-s3</artifactId>