Upgrade repository-s3 to AWS SDK v2

build-tools-internal/version.properties

@@ -17,7 +17,6 @@ jna               = 5.12.1
 netty             = 4.1.118.Final
 commons_lang3     = 3.9
 google_oauth_client = 1.34.1
-awsv1sdk            = 1.12.746
 awsv2sdk            = 2.30.38
 reactive_streams    = 1.0.4
 

docs/changelog/126843.yaml

@@ -0,0 +1,90 @@
+pr: 126843
+summary: Upgrade `repository-s3` to AWS SDK v2
+area: Snapshot/Restore
+type: breaking
+issues:
+ - 120993
+highlight:
+  title: Upgrade `repository-s3` to AWS SDK v2
+  body: >-
+    In earlier versions of {es} the `repository-s3` plugin was based on the AWS
+    SDK v1. AWS will withdraw support for this SDK before the end of the life
+    of {es} {minor-version} so we have migrated this plugin to the newer AWS SDK v2.
+
+    The two SDKs are not quite compatible, so please check the breaking changes
+    documentation and test the new version thoroughly before upgrading any
+    production workloads.
+  notable: true
+breaking:
+  title: Upgrade `repository-s3` to AWS SDK v2
+  area: Cluster and node setting
+  details: >-
+    In earlier versions of {es} the `repository-s3` plugin was based on the AWS
+    SDK v1. AWS will withdraw support for this SDK before the end of the life
+    of {es} {minor-version} so we must migrate to the newer AWS SDK v2.
+
+    Unfortunately there are several differences between the two AWS SDK
+    versions which may require you to adjust your system configuration when
+    upgrading to {es} {minor-version} or later. These differences include, but
+    may not be limited to, the following items.
+
+    * AWS SDK v2 requires users to specify the region to use for signing
+      requests, or else to run in an environment in which it can determine the
+      correct region automatically. The older SDK would try to determine the
+      region based on the endpoint URL as specified with the
+      `s3.client.${CLIENT_NAME}.endpoint` setting, together with other data
+      drawn from the operating environment, and would ultimately fall back to
+      `us-east-1` if no better value could be found.
+
+    * AWS SDK v2 does not support the EC2 IMDSv1 protocol.
+
+    * AWS SDK v2 does not support the
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
+
+    * AWS SDK v2 does not permit specifying a choice between HTTP and HTTPS so
+      the `s3.client.${CLIENT_NAME}.protocol` setting is deprecated and no longer
+      has any effect.
+
+    * AWS SDK v2 does not permit control over throttling for retries, so the
+      the `s3.client.${CLIENT_NAME}.use_throttle_retries` setting is deprecated
+      and no longer has any effect.
+
+    * AWS SDK v2 requires the use of the V4 signature algorithm, so the
+      `s3.client.${CLIENT_NAME}.signer_override` setting is deprecated and no
+      longer has any effect.
+
+    * AWS SDK v2 does not support the `log-delivery-write` canned ACL.
+
+    * AWS SDK v2 counts 4xx responses differently in its metrics reporting.
+
+    * AWS SDK v2 always uses the regional STS endpoint, whereas AWS SDK v2
+      could use either a regional endpoint or the global
+      `https://sts.amazonaws.com` one.
+
+  impact: >-
+    If you use the `repository-s3` module, test your upgrade thoroughly before
+    upgrading any production workloads.
+
+    Adapt your configuration to the new SDK functionality. This includes, but
+    may not be limited to, the following items.
+
+    * Specify the correct signing region using the
+      `s3.client.${CLIENT_NAME}.region` setting on each node. {es} will try and
+      determine the correct region based on the endpoint URL and other data
+      drawn from the operating environment but cannot guarantee to do so
+      correctly in all cases.
+
+    * If you use IMDS to determine the availability zone of a node or to obtain
+      credentials for accessing the EC2 API, ensure that it supports the IMDSv2
+      protocol.
+
+    * If applicable, discontinue use of the
+      `com.amazonaws.sdk.ec2MetadataServiceEndpointOverride` system property.
+
+    * If applicable, specify that you wish to use the insecure HTTP protocol to
+      access the S3 API by setting `s3.client.${CLIENT_NAME}.endpoint` to a URL
+      which starts with `http://`.
+
+    * If applicable, discontinue use of the `log-delivery-write` canned ACL.
+
+  notable: true

gradle/verification-metadata.xml

@@ -86,36 +86,6 @@
             <sha256 value="395eebdbfaae281244b2da7f369654d6ca79c4be70acfff87e6638c07f89a0be" origin="Generated by Gradle"/>
          </artifact>
       </component>
-      <component group="com.amazonaws" name="aws-java-sdk-bedrockruntime" version="1.12.740">
-         <artifact name="aws-java-sdk-bedrockruntime-1.12.740.jar">
-            <sha256 value="ccc7efe5cd3ce22d6046cafd4d2f8bff5adcb43e0d27da482178fac5daadef81" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.amazonaws" name="aws-java-sdk-core" version="1.12.746">
-         <artifact name="aws-java-sdk-core-1.12.746.jar">
-            <sha256 value="798fd30dafcf6816e760ad8aef8b3f09c43351ed2e166993bddc4527dbafb0be" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.amazonaws" name="aws-java-sdk-ec2" version="1.12.746">
-         <artifact name="aws-java-sdk-ec2-1.12.746.jar">
-            <sha256 value="cec22d57e05ed75417b1342e9dd468c6fe7f2fab97c626c065d6495e44d732ad" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.amazonaws" name="aws-java-sdk-s3" version="1.12.746">
-         <artifact name="aws-java-sdk-s3-1.12.746.jar">
-            <sha256 value="dcd839802c71ffc4d3e6bebc8769a2149bc423baf95f3e6c8214f9c91536bc38" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.amazonaws" name="aws-java-sdk-sts" version="1.12.746">
-         <artifact name="aws-java-sdk-sts-1.12.746.jar">
-            <sha256 value="2916c28f9a6b6ade40c7e2ffdea3788b198a98b2b16830e02a24ec49fc0fb06f" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
-      <component group="com.amazonaws" name="jmespath-java" version="1.12.746">
-         <artifact name="jmespath-java-1.12.746.jar">
-            <sha256 value="d4239a7a1bfacbb9cd1f0e48a46ac95960ab7942c6fbb41ea825161efea72351" origin="Generated by Gradle"/>
-         </artifact>
-      </component>
       <component group="com.avast.gradle" name="gradle-docker-compose-plugin" version="0.17.5">
          <artifact name="gradle-docker-compose-plugin-0.17.5.jar">
             <sha256 value="bc818ee3015f7cea73d5a603fc7a542ad82ebb5799e406d9abc81ac42caa90f0" origin="Generated by Gradle"/>
@@ -4792,6 +4762,11 @@
             <sha256 value="ebb1d3d05711ccf2aa9bfc43fcc69fbe32e7be69e006e7952679c2f37d149f4d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="arns" version="2.30.38">
+         <artifact name="arns-2.30.38.jar">
+            <sha256 value="8e2f30384e603bfe793932a143dc6d55fabaaefe85567d09ee6e29adce3892fe" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="auth" version="2.30.38">
          <artifact name="auth-2.30.38.jar">
             <sha256 value="22d59f9af8111be5219eb33ef480d84c616565913da57cb4eac686076fea370e" origin="Generated by Gradle"/>
@@ -4812,6 +4787,11 @@
             <sha256 value="bfd558e937de70c3260df2356b47a25b562c59b5ebeded6b199846cc9a354fe5" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="aws-xml-protocol" version="2.30.38">
+         <artifact name="aws-xml-protocol-2.30.38.jar">
+            <sha256 value="6940bdaaa0dd135a6389bcd51a7bc613cf040cb8a689b5db7dcc18443b33d1fe" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="bedrockruntime" version="2.30.38">
          <artifact name="bedrockruntime-2.30.38.jar">
             <sha256 value="4424437b49fdf263ea460f4da634d3279ada7f4763827d74fea48c0f8f2afea3" origin="Generated by Gradle"/>
@@ -4912,11 +4892,21 @@
             <sha256 value="da37cb021156b6aae5a30337e270a33a43817a64c59ca7aa4c39074cfda39a4b" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="s3" version="2.30.38">
+         <artifact name="s3-2.30.38.jar">
+            <sha256 value="c83dd82a9d82ff8c7d2eb1bdb2ae9f9505b312dad9a6bf0b80bc0136653a3a24" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="sdk-core" version="2.30.38">
          <artifact name="sdk-core-2.30.38.jar">
             <sha256 value="556463b8c353408d93feab74719d141fcfda7fd3d7b7d1ad3a8a548b7cc2982d" origin="Generated by Gradle"/>
          </artifact>
       </component>
+      <component group="software.amazon.awssdk" name="sts" version="2.30.38">
+         <artifact name="sts-2.30.38.jar">
+            <sha256 value="29a4eb10332893b17a59f81c9d5b3fbf5caa8a386479f9edf5e81b9b8961af63" origin="Generated by Gradle"/>
+         </artifact>
+      </component>
       <component group="software.amazon.awssdk" name="third-party-jackson-core" version="2.30.38">
          <artifact name="third-party-jackson-core-2.30.38.jar">
             <sha256 value="979215cd78fe0b4abfa7465e6400b29ed90ced24d76323e87b6717195f0214af" origin="Generated by Gradle"/>

modules/repository-s3/build.gradle

@@ -19,27 +19,49 @@ esplugin {
 }
 
 dependencies {
-  api "com.amazonaws:aws-java-sdk-s3:${versions.awsv1sdk}"
-  api "com.amazonaws:aws-java-sdk-core:${versions.awsv1sdk}"
-  api "com.amazonaws:aws-java-sdk-sts:${versions.awsv1sdk}"
-  api "com.amazonaws:jmespath-java:${versions.awsv1sdk}"
-  api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
-  api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
-  api "commons-logging:commons-logging:${versions.commonslogging}"
-  api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
-  api "commons-codec:commons-codec:${versions.commonscodec}"
-  api "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
-  api "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
-  api "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:${versions.jackson}"
-  api "joda-time:joda-time:2.10.14"
-
-  // HACK: javax.xml.bind was removed from default modules in java 9, so we pull the api in here,
-  // and whitelist this hack in JarHell
-  api 'javax.xml.bind:jaxb-api:2.2.2'
+  implementation "software.amazon.awssdk:annotations:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:apache-client:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:auth:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:aws-core:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:aws-xml-protocol:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:http-client-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:identity-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:metrics-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:regions:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:retries-spi:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:retries:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:s3:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:sdk-core:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:services:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:sts:${versions.awsv2sdk}"
+  implementation "software.amazon.awssdk:utils:${versions.awsv2sdk}"
+
+  implementation "org.apache.httpcomponents:httpclient:${versions.httpclient}"
+
+  runtimeOnly "commons-codec:commons-codec:${versions.commonscodec}"
+  runtimeOnly "commons-logging:commons-logging:${versions.commonslogging}"
+  runtimeOnly "joda-time:joda-time:2.10.14"
+  runtimeOnly "org.apache.httpcomponents:httpcore:${versions.httpcore}"
+  runtimeOnly "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
+  runtimeOnly "org.reactivestreams:reactive-streams:${versions.reactive_streams}"
+  runtimeOnly "org.slf4j:slf4j-api:${versions.slf4j}"
+  runtimeOnly "software.amazon.awssdk:arns:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:aws-query-protocol:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:checksums-spi:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:checksums:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:endpoints-spi:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:http-auth:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:http-auth-aws:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:http-auth-spi:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:json-utils:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:profiles:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:protocol-core:${versions.awsv2sdk}"
+  runtimeOnly "software.amazon.awssdk:third-party-jackson-core:${versions.awsv2sdk}"
 
   testImplementation project(':test:fixtures:s3-fixture')
+  testImplementation "software.amazon.awssdk:endpoints-spi:${versions.awsv2sdk}"
 
+  internalClusterTestImplementation project(':test:fixtures:aws-fixture-utils')
   internalClusterTestImplementation project(':test:fixtures:minio-fixture')
   internalClusterTestRuntimeOnly "org.slf4j:slf4j-simple:${versions.slf4j}"
 
@@ -68,10 +90,34 @@ restResources {
 }
 
 tasks.named("dependencyLicenses").configure {
-  mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
-  mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
-  mapping from: /jackson-.*/, to: 'jackson'
-  mapping from: /jaxb-.*/, to: 'jaxb'
+  mapping from: 'annotations',              to: 'aws-sdk-2'
+  mapping from: 'apache-client',            to: 'aws-sdk-2'
+  mapping from: 'arns',                     to: 'aws-sdk-2'
+  mapping from: 'auth',                     to: 'aws-sdk-2'
+  mapping from: 'aws-core',                 to: 'aws-sdk-2'
+  mapping from: 'aws-query-protocol',       to: 'aws-sdk-2'
+  mapping from: 'aws-xml-protocol',         to: 'aws-sdk-2'
+  mapping from: 'checksums',                to: 'aws-sdk-2'
+  mapping from: 'checksums-spi',            to: 'aws-sdk-2'
+  mapping from: 'endpoints-spi',            to: 'aws-sdk-2'
+  mapping from: 'http-auth',                to: 'aws-sdk-2'
+  mapping from: 'http-auth-aws',            to: 'aws-sdk-2'
+  mapping from: 'http-auth-spi',            to: 'aws-sdk-2'
+  mapping from: 'http-client-spi',          to: 'aws-sdk-2'
+  mapping from: 'identity-spi',             to: 'aws-sdk-2'
+  mapping from: 'json-utils',               to: 'aws-sdk-2'
+  mapping from: 'metrics-spi',              to: 'aws-sdk-2'
+  mapping from: 'profiles',                 to: 'aws-sdk-2'
+  mapping from: 'protocol-core',            to: 'aws-sdk-2'
+  mapping from: 'regions',                  to: 'aws-sdk-2'
+  mapping from: 'retries',                  to: 'aws-sdk-2'
+  mapping from: 'retries-spi',              to: 'aws-sdk-2'
+  mapping from: 's3',                       to: 'aws-sdk-2'
+  mapping from: 'sdk-core',                 to: 'aws-sdk-2'
+  mapping from: 'services',                 to: 'aws-sdk-2'
+  mapping from: 'sts',                      to: 'aws-sdk-2'
+  mapping from: 'third-party-jackson-core', to: 'aws-sdk-2'
+  mapping from: 'utils',                    to: 'aws-sdk-2'
 }
 
 esplugin.bundleSpec.from('config/repository-s3') {
@@ -85,23 +131,61 @@ tasks.named("internalClusterTest").configure {
 
 tasks.named("thirdPartyAudit").configure {
   ignoreMissingClasses(
-          // classes are missing
-          'javax.servlet.ServletContextEvent',
-          'javax.servlet.ServletContextListener',
-          'org.apache.avalon.framework.logger.Logger',
-          'org.apache.log.Hierarchy',
-          'org.apache.log.Logger',
-          'javax.jms.Message',
-          // We don't use the kms dependency
-          'com.amazonaws.services.kms.AWSKMS',
-          'com.amazonaws.services.kms.AWSKMSClient',
-          'com.amazonaws.services.kms.AWSKMSClientBuilder',
-          'com.amazonaws.services.kms.model.DecryptRequest',
-          'com.amazonaws.services.kms.model.DecryptResult',
-          'com.amazonaws.services.kms.model.EncryptRequest',
-          'com.amazonaws.services.kms.model.EncryptResult',
-          'com.amazonaws.services.kms.model.GenerateDataKeyRequest',
-          'com.amazonaws.services.kms.model.GenerateDataKeyResult',
-          'javax.activation.DataHandler'
+    // missing/unused classes
+    'javax.servlet.ServletContextEvent',
+    'javax.servlet.ServletContextListener',
+    'org.apache.avalon.framework.logger.Logger',
+    'org.apache.log.Hierarchy',
+    'org.apache.log.Logger',
+    'javax.jms.Message',
+
+    // We use the Apache HTTP client rather than an AWS common runtime (CRT) one, so we don't need any of these classes:
+    'software.amazon.awssdk.crt.CRT',
+    'software.amazon.awssdk.crt.auth.credentials.Credentials',
+    'software.amazon.awssdk.crt.auth.credentials.CredentialsProvider',
+    'software.amazon.awssdk.crt.auth.credentials.DelegateCredentialsProvider$DelegateCredentialsProviderBuilder',
+    'software.amazon.awssdk.crt.auth.signing.AwsSigner',
+    'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSignatureType',
+    'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSignedBodyHeaderType',
+    'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig$AwsSigningAlgorithm',
+    'software.amazon.awssdk.crt.auth.signing.AwsSigningConfig',
+    'software.amazon.awssdk.crt.auth.signing.AwsSigningResult',
+    'software.amazon.awssdk.crt.http.HttpHeader',
+    'software.amazon.awssdk.crt.http.HttpMonitoringOptions',
+    'software.amazon.awssdk.crt.http.HttpProxyEnvironmentVariableSetting$HttpProxyEnvironmentVariableType',
+    'software.amazon.awssdk.crt.http.HttpProxyEnvironmentVariableSetting',
+    'software.amazon.awssdk.crt.http.HttpProxyOptions',
+    'software.amazon.awssdk.crt.http.HttpRequest',
+    'software.amazon.awssdk.crt.http.HttpRequestBodyStream',
+    'software.amazon.awssdk.crt.io.ClientBootstrap',
+    'software.amazon.awssdk.crt.io.ExponentialBackoffRetryOptions',
+    'software.amazon.awssdk.crt.io.StandardRetryOptions',
+    'software.amazon.awssdk.crt.io.TlsCipherPreference',
+    'software.amazon.awssdk.crt.io.TlsContext',
+    'software.amazon.awssdk.crt.io.TlsContextOptions',
+    'software.amazon.awssdk.crt.s3.ChecksumAlgorithm',
+    'software.amazon.awssdk.crt.s3.ChecksumConfig$ChecksumLocation',
+    'software.amazon.awssdk.crt.s3.ChecksumConfig',
+    'software.amazon.awssdk.crt.s3.ResumeToken',
+    'software.amazon.awssdk.crt.s3.S3Client',
+    'software.amazon.awssdk.crt.s3.S3ClientOptions',
+    'software.amazon.awssdk.crt.s3.S3FinishedResponseContext',
+    'software.amazon.awssdk.crt.s3.S3MetaRequest',
+    'software.amazon.awssdk.crt.s3.S3MetaRequestOptions$MetaRequestType',
+    'software.amazon.awssdk.crt.s3.S3MetaRequestOptions',
+    'software.amazon.awssdk.crt.s3.S3MetaRequestProgress',
+    'software.amazon.awssdk.crt.s3.S3MetaRequestResponseHandler',
+    'software.amazon.awssdk.crtcore.CrtConfigurationUtils',
+    'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$Builder',
+    'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration$DefaultBuilder',
+    'software.amazon.awssdk.crtcore.CrtConnectionHealthConfiguration',
+    'software.amazon.awssdk.crtcore.CrtProxyConfiguration$Builder',
+    'software.amazon.awssdk.crtcore.CrtProxyConfiguration$DefaultBuilder',
+    'software.amazon.awssdk.crtcore.CrtProxyConfiguration',
+
+    // We don't use anything eventstream-based so these classes are not needed:
+    'software.amazon.eventstream.HeaderValue',
+    'software.amazon.eventstream.Message',
+    'software.amazon.eventstream.MessageDecoder'
   )
 }