Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
import software.amazon.awssdk.metrics.MetricPublisher;
import software.amazon.awssdk.services.s3.S3AsyncClient;
import software.amazon.awssdk.services.s3.S3IntegrationTestBase;
import software.amazon.awssdk.services.s3.model.ChecksumMode;
import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
Expand Down Expand Up @@ -280,6 +281,61 @@ void getObject_emptyObject_withRange_shouldThrow416() throws Exception {
.hasCauseInstanceOf(S3Exception.class);
}

@Test
void getObject_withChecksumModeEnabled_returnsChecksumHeaders() throws Exception {
PresignedGetObjectRequest presigned = presigner.presignGetObject(r -> r
.getObjectRequest(req -> req.bucket(testBucket).key(testGetObjectKey)
.checksumMode(ChecksumMode.ENABLED))
.signatureDuration(Duration.ofMinutes(10)));

PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder()
.presignedUrl(presigned.url())
.build();

ResponseBytes<GetObjectResponse> response =
presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes())
.get(30, TimeUnit.SECONDS);

assertThat(response.asUtf8String()).isEqualTo(testObjectContent);
assertThat(response.response().checksumTypeAsString()).isNotNull();
}

@Test
void getObject_withoutChecksumMode_doesNotReturnChecksumHeaders() throws Exception {
PresignedUrlDownloadRequest request = createRequestForKey(testGetObjectKey);

ResponseBytes<GetObjectResponse> response =
presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes())
.get(30, TimeUnit.SECONDS);

assertThat(response.asUtf8String()).isEqualTo(testObjectContent);
assertThat(response.response().checksumTypeAsString()).isNull();
}

@Test
void getObject_withChecksumModeEnabled_requestContainsChecksumHeader() throws Exception {
PresignedGetObjectRequest presigned = presigner.presignGetObject(r -> r
.getObjectRequest(req -> req.bucket(testBucket).key(testGetObjectKey)
.checksumMode(ChecksumMode.ENABLED))
.signatureDuration(Duration.ofMinutes(10)));

// Verify the presigned URL has checksum-mode in SignedHeaders
assertThat(presigned.signedHeaders()).containsKey("x-amz-checksum-mode");
assertThat(presigned.isBrowserExecutable()).isFalse();

PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder()
.presignedUrl(presigned.url())
.build();

// Download should succeed (proves marshaller sent the header)
ResponseBytes<GetObjectResponse> response =
presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes())
.get(30, TimeUnit.SECONDS);

assertThat(response).isNotNull();
assertThat(response.asUtf8String()).isEqualTo(testObjectContent);
}

static Stream<Arguments> basicFunctionalityTestData() {
return Stream.of(
Arguments.of("getObject_withValidUrl_returnsContent",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import software.amazon.awssdk.core.interceptor.SdkExecutionAttribute;
import software.amazon.awssdk.core.internal.util.HttpChecksumUtils;
import software.amazon.awssdk.http.SdkHttpResponse;
import software.amazon.awssdk.services.s3.internal.presignedurl.model.PresignedUrlDownloadRequestWrapper;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import software.amazon.awssdk.utils.Pair;
Expand All @@ -43,7 +44,8 @@ public class GetObjectInterceptor implements ExecutionInterceptor {
@Override
public void afterTransmission(Context.AfterTransmission context, ExecutionAttributes executionAttributes) {

if (!(context.request() instanceof GetObjectRequest)) {
if (!(context.request() instanceof GetObjectRequest)
&& !(context.request() instanceof PresignedUrlDownloadRequestWrapper)) {
return;
}
ChecksumSpecs resolvedChecksumSpecs = executionAttributes.getAttribute(SdkExecutionAttribute.RESOLVED_CHECKSUM_SPECS);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import software.amazon.awssdk.annotations.SdkInternalApi;
import software.amazon.awssdk.awscore.exception.AwsServiceException;
import software.amazon.awssdk.awscore.internal.AwsProtocolMetadata;
import software.amazon.awssdk.checksums.DefaultChecksumAlgorithm;
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
import software.amazon.awssdk.core.async.AsyncResponseTransformerUtils;
import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption;
Expand All @@ -37,6 +38,7 @@
import software.amazon.awssdk.core.exception.SdkClientException;
import software.amazon.awssdk.core.http.HttpResponseHandler;
import software.amazon.awssdk.core.interceptor.SdkInternalExecutionAttribute;
import software.amazon.awssdk.core.interceptor.trait.HttpChecksum;
import software.amazon.awssdk.core.metrics.CoreMetric;
import software.amazon.awssdk.core.signer.NoOpSigner;
import software.amazon.awssdk.metrics.MetricCollector;
Expand All @@ -61,12 +63,31 @@
public final class DefaultAsyncPresignedUrlExtension implements AsyncPresignedUrlExtension {
private static final Logger log = LoggerFactory.getLogger(DefaultAsyncPresignedUrlExtension.class);

/**
* Checksum configuration matching the codegen-produced GetObject operation.
* Enables the HttpChecksumValidationInterceptor to validate response checksums.
*/
private static final HttpChecksum RESPONSE_CHECKSUM_CONFIG = HttpChecksum.builder()
.requestValidationMode("ENABLED")
.responseAlgorithmsV2(
DefaultChecksumAlgorithm.XXHASH3,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm a little worried about the hard coding of these algorithms here. It would be easy for us to forget to update this with new algorithms.... is there any way we could create it dynamically?

DefaultChecksumAlgorithm.XXHASH128,
DefaultChecksumAlgorithm.CRC64NVME,
DefaultChecksumAlgorithm.CRC32C,
DefaultChecksumAlgorithm.CRC32,
DefaultChecksumAlgorithm.XXHASH64,
DefaultChecksumAlgorithm.SHA512,
DefaultChecksumAlgorithm.SHA256,
DefaultChecksumAlgorithm.SHA1,
DefaultChecksumAlgorithm.MD5)
.build();

private final AsyncClientHandler clientHandler;
private final AwsS3ProtocolFactory protocolFactory;
private final SdkClientConfiguration clientConfiguration;
private final List<MetricPublisher> metricPublishers;
private final AwsProtocolMetadata protocolMetadata;

public DefaultAsyncPresignedUrlExtension(AsyncClientHandler clientHandler,
AwsS3ProtocolFactory protocolFactory,
SdkClientConfiguration clientConfiguration,
Expand Down Expand Up @@ -122,6 +143,8 @@ public <ReturnT> CompletableFuture<ReturnT> getObject(
.withMetricCollector(apiCallMetricCollector)
// TODO: Deprecate IS_DISCOVERED_ENDPOINT, use new SKIP_ENDPOINT_RESOLUTION for better semantics
.putExecutionAttribute(SdkInternalExecutionAttribute.IS_DISCOVERED_ENDPOINT, true)
.putExecutionAttribute(SdkInternalExecutionAttribute.HTTP_CHECKSUM,
checksumConfigForUrl(presignedUrlDownloadRequest.presignedUrl()))
.withMarshaller(new PresignedUrlDownloadRequestMarshaller(protocolFactory)),
asyncResponseTransformer);

Expand Down Expand Up @@ -151,4 +174,15 @@ private SdkClientConfiguration updateSdkClientConfiguration(SdkClientConfigurati
return configBuilder.build();
}

/**
* Returns the checksum validation config if the presigned URL was signed with checksum mode,
* or null otherwise.
*/
private static HttpChecksum checksumConfigForUrl(java.net.URL presignedUrl) {
if (PresignedUrlDownloadRequestMarshaller.hasChecksumModeInSignedHeaders(presignedUrl.getQuery())) {
return RESPONSE_CHECKSUM_CONFIG;
}
return null;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -63,14 +63,43 @@ public SdkHttpFullRequest marshall(PresignedUrlDownloadRequestWrapper presignedU
.createProtocolMarshaller(SDK_OPERATION_BINDING);
URI presignedUri = presignedUrlDownloadRequestWrapper.url().toURI();

return protocolMarshaller.marshall(presignedUrlDownloadRequestWrapper)
.toBuilder()
.uri(presignedUri)
.build();
SdkHttpFullRequest.Builder requestBuilder = protocolMarshaller
.marshall(presignedUrlDownloadRequestWrapper)
.toBuilder()
.uri(presignedUri);

addChecksumModeHeaderIfSignedInUrl(requestBuilder, presignedUri);

return requestBuilder.build();
} catch (Exception e) {
throw SdkClientException.builder()
.message("Unable to marshall pre-signed URL Request: " + e.getMessage())
.cause(e).build();
}
}
}

/**
* If the presigned URL's X-Amz-SignedHeaders contains "x-amz-checksum-mode", automatically add
* the header with value "ENABLED" so S3 returns checksum headers in the response.
*/
private void addChecksumModeHeaderIfSignedInUrl(SdkHttpFullRequest.Builder requestBuilder, URI uri) {
if (hasChecksumModeInSignedHeaders(uri.getQuery())) {
requestBuilder.putHeader("x-amz-checksum-mode", "ENABLED");
}
}

/**
* Returns true if the decoded query string's X-Amz-SignedHeaders parameter contains "x-amz-checksum-mode".
*/
static boolean hasChecksumModeInSignedHeaders(String query) {
if (query == null) {
return false;
}
for (String param : query.split("&")) {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the query parameters here are URL encoded - I'm trying to think through if there are any cases where we would end up with something invalid because we're not actually decoding these first.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We use uri.getQuery() which returns the decoded query string, so parsing operates on decoded values.

if (param.startsWith("X-Amz-SignedHeaders=")) {
return param.substring("X-Amz-SignedHeaders=".length()).contains("x-amz-checksum-mode");

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can't remember exactly how all of the signing works, but are these guaranteed to be lower case?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, V4CanonicalRequest.getCanonicalHeaders() lowercases all header names per the SigV4 specification, so X-Amz-SignedHeaders always contains lowercase values.

}
}
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,14 @@
/**
* Interface for executing S3 operations asynchronously using presigned URLs. This can be accessed using
* {@link S3AsyncClient#presignedUrlExtension()}.
*
* <p><b>Checksum Validation:</b> If the presigned URL was generated with
* {@link software.amazon.awssdk.services.s3.presigner.S3Presigner#presignGetObject} using
* {@code checksumMode(ChecksumMode.ENABLED)}, the SDK automatically sends the required header
* and S3 returns checksums for full object downloads (HTTP 200). For ranged downloads (HTTP 206),
* checksums are only returned for multipart-uploaded objects when the range aligns with original
* upload part boundaries. The downloader cannot enable checksums if the URL was not presigned
* with checksum mode.
*/
@SdkPublicApi
public interface AsyncPresignedUrlExtension {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,12 @@ static Builder builder() {
* signing or authentication.
* <p/>
*
* <b>Checksum support:</b> Setting {@code checksumMode(ChecksumMode.ENABLED)} on the
* {@code GetObjectRequest} enables the downloader to receive checksums from S3 for data integrity
* validation. The resulting URL will not be browser-executable (requires the
* {@code x-amz-checksum-mode} header at download time, which the SDK sends automatically).
* <p/>
*
* <b>Example Usage</b>
* <p/>
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -341,6 +341,71 @@ private void assertSuccessfulGetObject(PresignedUrlDownloadRequest request) {
}
}

@ParameterizedTest(name = "{0}")
@MethodSource("checksumValidationTestCases")
void getObject_withChecksumHeaderInResponse_shouldDownloadSuccessfully(
String testName,
HttpExecuteResponse response,
String testUrl,
boolean expectSuccess) throws Exception {

mockHttpClient.stubNextResponse(response);

URL presignedUrl = new URL(testUrl);
PresignedUrlDownloadRequest request = PresignedUrlDownloadRequest.builder()
.presignedUrl(presignedUrl)
.build();

CompletableFuture<ResponseBytes<GetObjectResponse>> future =
presignedUrlExtension.getObject(request, AsyncResponseTransformer.toBytes());

ResponseBytes<GetObjectResponse> result = future.get();
assertThat(result).isNotNull();
assertThat(result.asUtf8String()).isEqualTo(TEST_CONTENT);
}

private static Stream<Arguments> checksumValidationTestCases() {
// CRC32 of "test-content" = 0x6B59FCDE → base64 = a1n83g==
String correctCrc32 = "a1n83g==";
String urlWithChecksumSigned = "https://test-bucket.s3.us-east-1.amazonaws.com/test-key?" +
"X-Amz-Algorithm=AWS4-HMAC-SHA256&" +
"X-Amz-Date=20250707T000000Z&" +
"X-Amz-SignedHeaders=host%3Bx-amz-checksum-mode&" +
"X-Amz-Signature=test-signature&" +
"X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20250707%2Fus-east-1%2Fs3%2Faws4_request&" +
"X-Amz-Expires=600";

return Stream.of(
Arguments.of(
"With checksum in response - should succeed",
createResponseWithChecksum("x-amz-checksum-crc32", correctCrc32),
urlWithChecksumSigned,
true
),
Arguments.of(
"No checksum header in response - should succeed without validation",
createSuccessResponse(),
TEST_URL,
true
)
);
}

private static HttpExecuteResponse createResponseWithChecksum(String checksumHeader, String checksumValue) {
SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder()
.statusCode(200)
.putHeader("Content-Length", "12")
.putHeader("ETag", "\"test-etag\"")
.putHeader("Content-Type", "text/plain")
.putHeader(checksumHeader, checksumValue)
.build();
return HttpExecuteResponse.builder()
.response(httpResponse)
.responseBody(AbortableInputStream.create(
new ByteArrayInputStream(TEST_CONTENT.getBytes(StandardCharsets.UTF_8))))
.build();
}

private static HttpExecuteResponse createSuccessResponse() {
SdkHttpFullResponse httpResponse = SdkHttpFullResponse.builder()
.statusCode(200)
Expand Down
Loading
Loading