Skip to content

Add support for Cloudflare R2 Storage #53

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
add AWS_ENDPOINT_URL env and fix get_object_attributes
  • Loading branch information
Spioune committed Mar 2, 2025
commit bf4067916748f7e39817a1113fe9606375ce8d8a
22 changes: 10 additions & 12 deletions crates/aws_s3/src/storage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ use async_trait::async_trait;
use aws_config::retry::RetryConfig;
use aws_sdk_s3::{
operation::{
get_object_attributes::{
GetObjectAttributesError,
GetObjectAttributesOutput,
head_object::{
HeadObjectOutput,
HeadObjectError,
},
upload_part::builders::UploadPartFluentBuilder,
},
Expand Down Expand Up @@ -434,26 +434,24 @@ impl<RT: Runtime> Storage for S3Storage<RT> {
.split_once('/')
.with_context(|| format!("Invalid fully qualified S3 key {:?}", key))?;
let result: Result<
GetObjectAttributesOutput,
aws_sdk_s3::error::SdkError<GetObjectAttributesError>,
HeadObjectOutput,
aws_sdk_s3::error::SdkError<HeadObjectError>,
> = self
.client
.get_object_attributes()
.head_object()
.bucket(bucket)
.key(s3_key)
.object_attributes(aws_sdk_s3::types::ObjectAttributes::Checksum)
.object_attributes(aws_sdk_s3::types::ObjectAttributes::ObjectSize)
.send()
.await;
match result {
Ok(object_attributes) => {
let size = object_attributes
.object_size
Ok(head_attributes) => {
let size = head_attributes
.content_length
.context("Object is missing size")? as u64;
Ok(Some(ObjectAttributes { size }))
},
Err(aws_sdk_s3::error::SdkError::ServiceError(err)) => match err.err() {
GetObjectAttributesError::NoSuchKey(_) => Ok(None),
HeadObjectError::NotFound(_) => Ok(None),
// Other service errors from S3
_ => Err(err.into_err().into()),
},
Expand Down
7 changes: 7 additions & 0 deletions crates/aws_utils/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ use aws_types::region::Region;

pub mod s3;

static AWS_ENDPOINT_URL: LazyLock<Option<String>> =
LazyLock::new(|| env::var("AWS_ENDPOINT_URL").ok());

static AWS_ACCESS_KEY_ID: LazyLock<Option<String>> =
LazyLock::new(|| env::var("AWS_ACCESS_KEY_ID").ok());

Expand All @@ -36,8 +39,12 @@ pub fn must_config_from_env() -> anyhow::Result<ConfigLoader> {
let Some(_) = AWS_SECRET_ACCESS_KEY.clone() else {
anyhow::bail!("AWS_SECRET_ACCESS_KEY env variable must be set");
};
let Some(endpoint_url) = AWS_ENDPOINT_URL.clone() else {
anyhow::bail!("AWS_ENDPOINT_URL env variable must be set");
};
let credentials = EnvironmentVariableCredentialsProvider::new();
Ok(aws_config::defaults(BehaviorVersion::v2024_03_28())
.endpoint_url(endpoint_url)
.region(region)
.credentials_provider(credentials))
}