diff --git a/cmake/sdksCommon.cmake b/cmake/sdksCommon.cmake index fc3bcc18023..26a2c119eb1 100644 --- a/cmake/sdksCommon.cmake +++ b/cmake/sdksCommon.cmake @@ -108,6 +108,7 @@ list(APPEND SDK_TEST_PROJECT_LIST "redshift:tests/aws-cpp-sdk-redshift-integrati list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-integration-tests") list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-unit-tests") list(APPEND SDK_TEST_PROJECT_LIST "s3-crt:tests/aws-cpp-sdk-s3-crt-integration-tests") +list(APPEND SDK_TEST_PROJECT_LIST "s3-transfer:tests/aws-cpp-sdk-s3-transfer-integration-tests") list(APPEND SDK_TEST_PROJECT_LIST "s3-encryption:tests/aws-cpp-sdk-s3-encryption-tests,tests/aws-cpp-sdk-s3-encryption-integration-tests") list(APPEND SDK_TEST_PROJECT_LIST "s3control:tests/aws-cpp-sdk-s3control-integration-tests") list(APPEND SDK_TEST_PROJECT_LIST "sns:tests/aws-cpp-sdk-sns-integration-tests") diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadDataReceiver.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadDataReceiver.h new file mode 100644 index 00000000000..74bb0ac17e8 --- /dev/null +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadDataReceiver.h @@ -0,0 +1,26 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once +#include +#include + +namespace Aws { +namespace S3 { +namespace Transfer { + +/** + * Callback interface for zero-copy downloads. The transfer manager delivers each part of the + * object to OnDataReceived as it arrives, in object order. Keep a copy of the buffer to retain + * the bytes past this call. + */ +class AWS_S3_TRANSFER_API DownloadDataReceiver { + public: + virtual ~DownloadDataReceiver() = default; + virtual void OnDataReceived(S3DownloadBuffer buffer) = 0; +}; + +} +} +} diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadHandle.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadHandle.h index 0bd91d7a1ca..5cc41532ea0 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadHandle.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadHandle.h @@ -1,5 +1,5 @@ /** -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ #pragma once @@ -7,33 +7,33 @@ #include #include #include +#include namespace Aws { namespace S3 { namespace Transfer { -/** - * Returned from S3TransferManager::Download to represent a single in-flight download. The - * handle is move-only and owns the underlying transfer state. - */ +class DownloadHandleImpl; + +// Move-only handle for a single in-flight download. class AWS_S3_TRANSFER_API DownloadHandle final { public: - DownloadHandle(); + explicit DownloadHandle(Aws::UniquePtr impl); ~DownloadHandle(); + DownloadHandle(const DownloadHandle&) = delete; + DownloadHandle& operator=(const DownloadHandle&) = delete; DownloadHandle(DownloadHandle&&) noexcept; DownloadHandle& operator=(DownloadHandle&&) noexcept; - /** - * Returns a future that resolves once the transfer finishes, succeeds, or fails. - */ + // Resolves once the transfer finishes, succeeds, or fails. std::future CompletionFuture(); - /** - * Requests cancellation of the in-flight download. Returns immediately; the future - * returned by CompletionFuture will resolve with a failure once the cancel takes effect. - */ + // Returns immediately; the completion future resolves with a failure once the cancel takes effect. void Cancel(); + +private: + Aws::UniquePtr m_impl; }; diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadRequest.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadRequest.h index 69eb938e2d5..f582c88382f 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadRequest.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadRequest.h @@ -5,10 +5,11 @@ #pragma once #include #include +#include #include #include -#include #include +#include #include #include @@ -17,25 +18,36 @@ namespace S3 { namespace Transfer { /** - * Request type for S3TransferManager::Download. Carries the inner S3 GetObjectRequest along - * with the local destination (file path or stream factory) and any request-level progress - * listeners. The transfer manager parallelizes large objects via ranged GETs internally. + * Request type for S3TransferManager::Download. The destination is either a file path or a + * zero-copy DownloadDataReceiver; the two constructors make that choice exclusive. */ class AWS_S3_TRANSFER_API DownloadRequest final { public: + // File download. explicit DownloadRequest( Aws::S3::Model::GetObjectRequest s3Request, Aws::String destinationFilePath, - Aws::IOStreamFactory responseStreamFactory, Aws::Vector> transferListeners = {}) : m_s3Request(std::move(s3Request)), m_destinationFilePath(std::move(destinationFilePath)), - m_responseStreamFactory(std::move(responseStreamFactory)), - m_transferListeners(std::move(transferListeners)) {} + m_transferListeners(std::move(transferListeners)) { + assert(!m_destinationFilePath.empty() && "DownloadRequest destination file path must not be empty"); + } + + // Zero-copy stream download. Each part is delivered to the receiver in object order. + explicit DownloadRequest( + Aws::S3::Model::GetObjectRequest s3Request, + std::shared_ptr dataReceiver, + Aws::Vector> transferListeners = {}) + : m_s3Request(std::move(s3Request)), + m_dataReceiver(std::move(dataReceiver)), + m_transferListeners(std::move(transferListeners)) { + assert(m_dataReceiver && "DownloadRequest data receiver must not be null"); + } inline const Aws::S3::Model::GetObjectRequest& GetS3Request() const { return m_s3Request; } inline const Aws::String& GetDestinationFilePath() const { return m_destinationFilePath; } - inline const Aws::IOStreamFactory& GetResponseStreamFactory() const { return m_responseStreamFactory; } + inline const std::shared_ptr& GetDownloadDataReceiver() const { return m_dataReceiver; } inline const Aws::Vector>& GetTransferListeners() const { return m_transferListeners; } @@ -44,7 +56,7 @@ class AWS_S3_TRANSFER_API DownloadRequest final { private: Aws::S3::Model::GetObjectRequest m_s3Request; Aws::String m_destinationFilePath; - Aws::IOStreamFactory m_responseStreamFactory; + std::shared_ptr m_dataReceiver; Aws::Vector> m_transferListeners; }; diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadResponse.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadResponse.h index 8aa84699b28..e719e9de722 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadResponse.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadResponse.h @@ -28,11 +28,6 @@ class AWS_S3_TRANSFER_API DownloadResponse final { m_s3ResultHasBeenSet = true; m_s3Result = std::forward(getS3Result); } - template - DownloadResponse& WithS3Result(GetObjectResultT&& getS3Result) { - SetS3Result(std::forward(getS3Result)); - return *this; - } private: Aws::S3::Model::GetObjectResult m_s3Result; diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3DownloadBuffer.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3DownloadBuffer.h new file mode 100644 index 00000000000..b4ff1ffba6e --- /dev/null +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3DownloadBuffer.h @@ -0,0 +1,46 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once +#include +#include +#include +#include + +namespace Aws { +namespace S3 { +namespace Transfer { + +/** + * One part of a zero-copy download, delivered to DownloadDataReceiver::OnDataReceived. GetData() + * returns a cursor into CRT-owned memory; this buffer holds an owning reference so the bytes stay + * valid as long as this object (or a copy of it) is alive. + */ +class AWS_S3_TRANSFER_API S3DownloadBuffer final { + public: + // The cursor carries the per-part body delivered by the CRT; the ticket is an owning reference + // that keeps the underlying CRT-owned buffer alive for as long as this object (or a copy) is alive. + explicit S3DownloadBuffer(std::shared_ptr ticket, + Aws::Crt::ByteCursor bytes, uint64_t rangeStart) noexcept; + + S3DownloadBuffer(const S3DownloadBuffer&) = default; + S3DownloadBuffer& operator=(const S3DownloadBuffer&) = default; + S3DownloadBuffer(S3DownloadBuffer&& other) noexcept; + S3DownloadBuffer& operator=(S3DownloadBuffer&& other) noexcept; + ~S3DownloadBuffer() = default; + + // Cursor into CRT-owned memory; empty if the part carried no bytes. + inline Aws::Crt::ByteCursor GetData() const { return m_bytes; } + // Byte offset within the object at which this part begins. + inline uint64_t GetRangeStart() const { return m_rangeStart; } + + private: + std::shared_ptr m_ticket; + Aws::Crt::ByteCursor m_bytes; + uint64_t m_rangeStart; +}; + +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManager.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManager.h index 52c5b03ae46..b29e1d93887 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManager.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManager.h @@ -1,5 +1,5 @@ /** -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ #pragma once @@ -11,41 +11,44 @@ #include #include namespace Aws { - namespace S3 { - namespace Transfer { - class S3TransferManagerImpl; +namespace S3 { +namespace Transfer { + +class S3TransferManagerImpl; + +/** + * Customers construct an instance directly. The manager owns the underlying CRT client; it is + * neither copyable nor movable. + */ +class AWS_S3_TRANSFER_API S3TransferManager final { + public: + explicit S3TransferManager(const S3TransferManagerConfiguration& config); + ~S3TransferManager(); + + S3TransferManager(const S3TransferManager&) = delete; + S3TransferManager& operator=(const S3TransferManager&) = delete; + S3TransferManager(S3TransferManager&&) noexcept = delete; + S3TransferManager& operator=(S3TransferManager&&) noexcept = delete; /** - * Customers construct an instance directly. The manager owns the underlying CRT - * client; it is movable but not copyable. + * Begin uploading the object described by request. Returns immediately with a handle that can be + * used to wait for completion or to cancel the in-flight transfer. */ - class AWS_S3_TRANSFER_API S3TransferManager final { - public: - explicit S3TransferManager(const S3TransferManagerConfiguration& config); - ~S3TransferManager(); - - S3TransferManager(const S3TransferManager&) = delete; - S3TransferManager& operator=(const S3TransferManager&) = delete; - - - /** - * Begin uploading the object described by request. Returns immediately with a handle - * that can be used to wait for completion or to cancel the in-flight transfer. - */ - UploadHandle Upload(const UploadRequest& request); - - /** - * Begin downloading the object described by request. Returns immediately with a handle - * that can be used to wait for completion or to cancel the in-flight transfer. - */ - DownloadHandle Download(const DownloadRequest& request); - - private: - Aws::UniquePtr m_impl; - }; - } - } - } + UploadHandle Upload(const UploadRequest& request); + + /** + * Begin downloading the object described by request. Returns immediately with a handle that can + * be used to wait for completion or to cancel the in-flight transfer. + */ + DownloadHandle Download(const DownloadRequest& request); + + private: + Aws::UniquePtr m_impl; +}; + +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManagerConfiguration.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManagerConfiguration.h index 0b37fa2f355..e46f83aca81 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManagerConfiguration.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManagerConfiguration.h @@ -7,20 +7,31 @@ #include #include +#include +#include +#include namespace Aws { namespace S3 { namespace Transfer { -struct AWS_S3_TRANSFER_API S3TransferManagerConfiguration : public Aws::Client::GenericClientConfiguration { - using BaseClientConfigClass = Aws::Client::GenericClientConfiguration; - - S3TransferManagerConfiguration(const Aws::Client::ClientConfigurationInitValues& configuration = {}); - S3TransferManagerConfiguration(const char* profileName, bool shouldDisableIMDS = false); +constexpr uint64_t DEFAULT_PART_SIZE_BYTES = 8ULL * 1024 * 1024; +constexpr uint64_t DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES = 16ULL * 1024 * 1024; - S3TransferManagerConfiguration(bool useSmartDefaults, const char* defaultMode = "legacy", bool shouldDisableIMDS = false); +struct AWS_S3_TRANSFER_API S3TransferManagerConfiguration final : public Aws::Client::GenericClientConfiguration { + using BaseClientConfigClass = Aws::Client::GenericClientConfiguration; - S3TransferManagerConfiguration(const Aws::Client::ClientConfiguration& config); + explicit S3TransferManagerConfiguration(const Aws::Client::ClientConfigurationInitValues& configuration = {}); + explicit S3TransferManagerConfiguration(const char* profileName, bool shouldDisableIMDS = false); + explicit S3TransferManagerConfiguration(bool useSmartDefaults, const char* defaultMode = "legacy", bool shouldDisableIMDS = false); + explicit S3TransferManagerConfiguration(const Aws::Client::ClientConfiguration& config); + + uint64_t partSize = DEFAULT_PART_SIZE_BYTES; + // Object size at or above which a multipart transfer is used. Must be >= partSize. + uint64_t multipartUploadThreshold = DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES; + // 0 means inherit the CRT's own default. + double throughputTargetGbps = 0.0; + Aws::Crt::Optional tlsConnectionOptions; private: void LoadS3TransferManagerSpecificConfig(const Aws::String& profileName); diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3Transfer_EXPORTS.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3Transfer_EXPORTS.h index fe7636bb906..8972c439bca 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3Transfer_EXPORTS.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3Transfer_EXPORTS.h @@ -1,5 +1,5 @@ /** -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ #pragma once diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadHandle.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadHandle.h index 804c6598750..08d17ce35bb 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadHandle.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadHandle.h @@ -1,5 +1,5 @@ /** -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ #pragma once @@ -7,34 +7,33 @@ #include #include #include +#include namespace Aws { namespace S3 { namespace Transfer { -/** - * Returned from S3TransferManager::Upload to represent a single in-flight upload. The - * handle is move-only and owns the underlying transfer state. - */ +class UploadHandleImpl; + +// Move-only handle for a single in-flight upload. class AWS_S3_TRANSFER_API UploadHandle final { public: - UploadHandle(); + explicit UploadHandle(Aws::UniquePtr impl); ~UploadHandle(); + UploadHandle(const UploadHandle&) = delete; + UploadHandle& operator=(const UploadHandle&) = delete; UploadHandle(UploadHandle&&) noexcept; UploadHandle& operator=(UploadHandle&&) noexcept; - - /** - * Returns a future that resolves once the transfer finishes, succeeds, or fails. - */ + // Resolves once the transfer finishes, succeeds, or fails. std::future CompletionFuture(); - /** - * Requests cancellation of the in-flight upload. Returns immediately; the future - * returned by CompletionFuture will resolve with a failure once the cancel takes effect. - */ + // Returns immediately; the completion future resolves with a failure once the cancel takes effect. void Cancel(); + +private: + Aws::UniquePtr m_impl; }; diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadRequest.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadRequest.h index b284a7bead1..5d76feb010e 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadRequest.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadRequest.h @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -16,36 +17,56 @@ namespace S3 { namespace Transfer { /** - * Request type for S3TransferManager::Upload. Carries the inner S3 PutObjectRequest along - * with the local source (file path or stream) and any request-level progress listeners. - * The transfer manager chooses between a single PutObject and a multipart upload based on - * the configured threshold. + * Request type for S3TransferManager::Upload. The upload source is either a file path or a stream + * body; the two constructors make that choice exclusive. */ class AWS_S3_TRANSFER_API UploadRequest final { public: + // File upload. explicit UploadRequest( Aws::S3::Model::PutObjectRequest s3Request, Aws::String sourceFilePath, - std::shared_ptr body, Aws::Vector> transferListeners = {}) : m_s3Request(std::move(s3Request)), m_sourceFilePath(std::move(sourceFilePath)), - m_body(std::move(body)), - m_transferListeners(std::move(transferListeners)) {} + m_transferListeners(std::move(transferListeners)) { + assert(!m_sourceFilePath.empty() && "UploadRequest file source path must not be empty"); + } - inline const Aws::S3::Model::PutObjectRequest& GetS3Request() const {return m_s3Request; } - inline const Aws::String& GetSourceFilePath() const {return m_sourceFilePath;} - inline const std::shared_ptr& GetBody() const {return m_body;} + // Stream upload. For a non-seekable stream, call SetContentLength if the length is known. + explicit UploadRequest( + Aws::S3::Model::PutObjectRequest s3Request, + std::shared_ptr body, + Aws::Vector> transferListeners = {}) + : m_s3Request(std::move(s3Request)), + m_transferListeners(std::move(transferListeners)) { + assert(body && "UploadRequest stream body must not be null"); + m_s3Request.SetBody(body); + } + + inline const Aws::S3::Model::PutObjectRequest& GetS3Request() const { return m_s3Request; } + inline const Aws::String& GetSourceFilePath() const { return m_sourceFilePath; } + inline std::shared_ptr GetBody() const { return m_s3Request.GetBody(); } inline const Aws::Vector>& GetTransferListeners() const { return m_transferListeners; } + // Content length of the stream body, in bytes. Only meaningful for stream uploads: supply it + // when the body is not seekable so the transfer manager need not measure the stream. + inline uint64_t GetContentLength() const { return m_contentLength; } + inline bool ContentLengthHasBeenSet() const { return m_contentLengthHasBeenSet; } + inline void SetContentLength(uint64_t contentLength) { + m_contentLength = contentLength; + m_contentLengthHasBeenSet = true; + } + private: Aws::S3::Model::PutObjectRequest m_s3Request; Aws::String m_sourceFilePath; - std::shared_ptr m_body; Aws::Vector> m_transferListeners; + uint64_t m_contentLength = 0; + bool m_contentLengthHasBeenSet = false; }; } diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadResponse.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadResponse.h index e5618229def..0ec2d037876 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadResponse.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadResponse.h @@ -28,11 +28,6 @@ class AWS_S3_TRANSFER_API UploadResponse final { m_s3ResultHasBeenSet = true; m_s3Result = std::forward(s3Result); } - template - UploadResponse& WithS3Result(S3ResultT&& s3Result) { - SetS3Result(std::forward(s3Result)); - return *this; - } private: Aws::S3::Model::PutObjectResult m_s3Result; diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/CrtOperations.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/CrtOperations.h index b45a188732f..2c23251e8f5 100644 --- a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/CrtOperations.h +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/CrtOperations.h @@ -18,11 +18,10 @@ class S3TransferManagerImpl; namespace Internal { /** - * Bridges TM 2.0 to the aws-crt-cpp S3 wrapper. Stays in C++; does not call - * aws-c-s3 directly. Internal; not part of the public API. - */ + * Bridges TM 2.0 to the aws-crt-cpp S3 wrapper. Stays in C++; does not call + * aws-c-s3 directly. Internal; not part of the public API. + */ class CrtOperations final { - public: /** * Dispatch an upload as a CRT meta request. Returns a handle bound to the diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/HandleImpls.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/HandleImpls.h new file mode 100644 index 00000000000..5039f227472 --- /dev/null +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/HandleImpls.h @@ -0,0 +1,35 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include + +namespace Aws { +namespace S3 { +namespace Transfer { + +template +class TransferHandleImpl { + public: + virtual ~TransferHandleImpl() = default; + + std::future future; + std::shared_ptr state; + + TransferHandleImpl() = default; + TransferHandleImpl(const TransferHandleImpl&) = delete; + TransferHandleImpl& operator=(const TransferHandleImpl&) = delete; + TransferHandleImpl(TransferHandleImpl&&) noexcept = default; + TransferHandleImpl& operator=(TransferHandleImpl&&) noexcept = default; +}; + +class UploadHandleImpl final : public TransferHandleImpl {}; +class DownloadHandleImpl final : public TransferHandleImpl {}; + +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/S3TransferManagerImpl.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/S3TransferManagerImpl.h new file mode 100644 index 00000000000..023a077352e --- /dev/null +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/S3TransferManagerImpl.h @@ -0,0 +1,54 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Aws { +namespace S3 { +namespace Transfer { + +constexpr size_t DEFAULT_EXECUTOR_POOL_SIZE = 8; + +// Precondition: Aws::InitAPI() must be called before construction. +class S3TransferManagerImpl final { + public: + explicit S3TransferManagerImpl(const S3TransferManagerConfiguration& config); + ~S3TransferManagerImpl(); + + S3TransferManagerImpl(const S3TransferManagerImpl&) = delete; + S3TransferManagerImpl& operator=(const S3TransferManagerImpl&) = delete; + S3TransferManagerImpl(S3TransferManagerImpl&&) = delete; + S3TransferManagerImpl& operator=(S3TransferManagerImpl&&) = delete; + + Aws::Crt::S3::S3Client& GetCrtClient() const { return *m_crtClient; } + Aws::S3::Endpoint::S3EndpointProvider& GetEndpointProvider() const { return *m_endpointProvider; } + const Aws::String& GetRegion() const { return m_region; } + const std::shared_ptr& GetCredentialsProvider() const { + return m_credentialsProvider; + } + bool IsValid() const { return m_crtClient != nullptr && static_cast(*m_crtClient); } + Aws::Utils::Threading::Executor& GetExecutor() const { return *m_executor; } + + private: + Aws::String m_region; + std::shared_ptr m_credentialsProvider; + Aws::UniquePtr m_endpointProvider; + Aws::UniquePtr m_crtClient; + std::shared_ptr m_executor; +}; + +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/TransferState.h b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/TransferState.h new file mode 100644 index 00000000000..cca5813f787 --- /dev/null +++ b/src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/internal/TransferState.h @@ -0,0 +1,79 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Aws { +namespace S3 { +namespace Transfer { + +// Per-transfer state shared by the CRT callbacks and the customer's handle. The promise is +// fulfilled exactly once, inside the finish callback. PublishMetaRequest and CancelMetaRequest +// synchronize through m_metaRequestLock so a cancel that races publication is never lost. +template +struct TransferStateBase { + std::promise promise; + RequestT request; + uint64_t totalBytes = 0; + bool totalBytesHasBeenSet = false; + std::atomic transferredBytes{0}; + Aws::Http::HeaderValueCollection responseHeaders; + int responseStatus = 0; + std::atomic canceled{false}; + + explicit TransferStateBase(const RequestT& req) : request(req) {} + virtual ~TransferStateBase() = default; + + TransferStateBase(const TransferStateBase&) = delete; + TransferStateBase& operator=(const TransferStateBase&) = delete; + TransferStateBase(TransferStateBase&&) = delete; + TransferStateBase& operator=(TransferStateBase&&) = delete; + + void PublishMetaRequest(std::shared_ptr metaRequest) { + std::lock_guard lock(m_metaRequestLock); + m_metaRequest = std::move(metaRequest); + if (canceled.load() && m_metaRequest) { + m_metaRequest->Cancel(); + } + } + + void CancelMetaRequest() { + std::lock_guard lock(m_metaRequestLock); + if (m_metaRequest) { + m_metaRequest->Cancel(); + } + } + + private: + mutable std::mutex m_metaRequestLock; + std::shared_ptr m_metaRequest; +}; + +struct UploadTransferState : TransferStateBase { + using TransferStateBase::TransferStateBase; +}; + +struct DownloadTransferState : TransferStateBase { + using TransferStateBase::TransferStateBase; + Aws::String destinationFilePath; + Aws::String tempFilePath; +}; + +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/DownloadHandle.cpp b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/DownloadHandle.cpp index e8189b20e69..d503abc779e 100644 --- a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/DownloadHandle.cpp +++ b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/DownloadHandle.cpp @@ -1,26 +1,35 @@ /** -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - #include + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include - namespace Aws { - namespace S3 { - namespace Transfer { - class DownloadHandleImpl { +namespace Aws { +namespace S3 { +namespace Transfer { - }; - DownloadHandle::~DownloadHandle() = default; - DownloadHandle::DownloadHandle() = default; - DownloadHandle::DownloadHandle(DownloadHandle&&) noexcept = default; - DownloadHandle& DownloadHandle::operator=(DownloadHandle&&) noexcept = default; +DownloadHandle::DownloadHandle(Aws::UniquePtr impl) : m_impl(std::move(impl)) {} - std::future DownloadHandle::CompletionFuture() { +DownloadHandle::~DownloadHandle() = default; + +DownloadHandle::DownloadHandle(DownloadHandle&&) noexcept = default; +DownloadHandle& DownloadHandle::operator=(DownloadHandle&&) noexcept = default; + +std::future DownloadHandle::CompletionFuture() { + if (!m_impl) { return {}; } - void DownloadHandle::Cancel() { + return std::move(m_impl->future); +} +void DownloadHandle::Cancel() { + if (m_impl && m_impl->state) { + m_impl->state->canceled.store(true); + m_impl->state->CancelMetaRequest(); } - } - } - } +} + +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/S3DownloadBuffer.cpp b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/S3DownloadBuffer.cpp new file mode 100644 index 00000000000..dcdc1542f80 --- /dev/null +++ b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/S3DownloadBuffer.cpp @@ -0,0 +1,33 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include + +namespace Aws { +namespace S3 { +namespace Transfer { + +S3DownloadBuffer::S3DownloadBuffer(std::shared_ptr ticket, + Aws::Crt::ByteCursor bytes, uint64_t rangeStart) noexcept + : m_ticket(std::move(ticket)), m_bytes(bytes), m_rangeStart(rangeStart) {} + +S3DownloadBuffer::S3DownloadBuffer(S3DownloadBuffer&& other) noexcept + : m_ticket(std::move(other.m_ticket)), m_bytes(other.m_bytes), m_rangeStart(other.m_rangeStart) { + other.m_bytes = Aws::Crt::ByteCursor{0, nullptr}; +} + +S3DownloadBuffer& S3DownloadBuffer::operator=(S3DownloadBuffer&& other) noexcept { + if (this != &other) { + m_ticket = std::move(other.m_ticket); + m_bytes = other.m_bytes; + m_rangeStart = other.m_rangeStart; + other.m_bytes = Aws::Crt::ByteCursor{0, nullptr}; + } + return *this; +} + +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/S3TransferManager.cpp b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/S3TransferManager.cpp index baf1d513aa5..40460b7c425 100644 --- a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/S3TransferManager.cpp +++ b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/S3TransferManager.cpp @@ -1,31 +1,29 @@ /** -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ #include +#include +#include #include namespace Aws { namespace S3 { namespace Transfer { -class S3TransferManagerImpl { -}; S3TransferManager::~S3TransferManager() = default; -S3TransferManager::S3TransferManager(const S3TransferManagerConfiguration& ) : -m_impl(Aws::MakeUnique("S3TransferManager")) { +S3TransferManager::S3TransferManager(const S3TransferManagerConfiguration& config) + : m_impl(Aws::MakeUnique("S3TransferManager", config)) {} +UploadHandle S3TransferManager::Upload(const UploadRequest& request) { + return Internal::CrtOperations::DispatchUpload(*m_impl, request); } -UploadHandle S3TransferManager::Upload(const UploadRequest&) { - return UploadHandle(); +DownloadHandle S3TransferManager::Download(const DownloadRequest& request) { + return Internal::CrtOperations::DispatchDownload(*m_impl, request); } -DownloadHandle S3TransferManager::Download(const DownloadRequest&) { - return DownloadHandle(); -} - -} -} -} +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/UploadHandle.cpp b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/UploadHandle.cpp index 2d46c4d976a..cc8f152755e 100644 --- a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/UploadHandle.cpp +++ b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/UploadHandle.cpp @@ -1,27 +1,36 @@ /** -* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - #include + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include +#include - namespace Aws { - namespace S3 { - namespace Transfer { - class UploadHandleImpl { +namespace Aws { +namespace S3 { +namespace Transfer { - }; - UploadHandle::~UploadHandle() = default; - UploadHandle::UploadHandle() = default; +UploadHandle::UploadHandle(Aws::UniquePtr impl) : m_impl(std::move(impl)) {} - UploadHandle::UploadHandle(UploadHandle&&) noexcept = default; - UploadHandle& UploadHandle::operator=(UploadHandle&&) noexcept = default; +UploadHandle::~UploadHandle() = default; - std::future UploadHandle::CompletionFuture() { +UploadHandle::UploadHandle(UploadHandle&&) noexcept = default; +UploadHandle& UploadHandle::operator=(UploadHandle&&) noexcept = default; + +std::future UploadHandle::CompletionFuture() { + if (!m_impl) { return {}; } - void UploadHandle::Cancel() { + return std::move(m_impl->future); +} +void UploadHandle::Cancel() { + if (m_impl && m_impl->state) { + // Signal the async-writes driver (if running) to stop pushing chunks before cancelling. + m_impl->state->canceled.store(true); + m_impl->state->CancelMetaRequest(); } - } - } - } +} + +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/internal/CrtOperations.cpp b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/internal/CrtOperations.cpp index 6474dfc6802..e3f92e2f5be 100644 --- a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/internal/CrtOperations.cpp +++ b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/internal/CrtOperations.cpp @@ -4,21 +4,729 @@ */ #include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include namespace Aws { namespace S3 { namespace Transfer { namespace Internal { -UploadHandle CrtOperations::DispatchUpload(S3TransferManagerImpl&, const UploadRequest&) { - return UploadHandle(); +static const char* const CRT_OPERATIONS_LOG_TAG = "CrtOperations"; + +namespace { + +Aws::Client::CoreErrors MapCrtErrorCode(int crtErrorCode) { + switch (crtErrorCode) { + case AWS_ERROR_S3_MISSING_CONTENT_RANGE_HEADER: + case AWS_ERROR_S3_MISSING_CONTENT_LENGTH_HEADER: + case AWS_ERROR_S3_MISSING_ETAG: + case AWS_ERROR_S3_MISSING_UPLOAD_ID: + return Aws::Client::CoreErrors::MISSING_PARAMETER; + case AWS_ERROR_S3_INVALID_CONTENT_RANGE_HEADER: + case AWS_ERROR_S3_INVALID_CONTENT_LENGTH_HEADER: + case AWS_ERROR_S3_INVALID_RANGE_HEADER: + case AWS_ERROR_S3_MULTIRANGE_HEADER_UNSUPPORTED: + case AWS_ERROR_S3_INCORRECT_CONTENT_LENGTH: + case AWS_ERROR_S3_INVALID_MEMORY_LIMIT_CONFIG: + return Aws::Client::CoreErrors::INVALID_PARAMETER_VALUE; + case AWS_ERROR_S3_INTERNAL_ERROR: + case AWS_ERROR_S3_PROXY_PARSE_FAILED: + case AWS_ERROR_S3_UNSUPPORTED_PROXY_SCHEME: + case AWS_ERROR_S3_NON_RECOVERABLE_ASYNC_ERROR: + case AWS_ERROR_S3_METRIC_DATA_NOT_AVAILABLE: + case AWS_ERROR_S3_EXCEEDS_MEMORY_LIMIT: + return Aws::Client::CoreErrors::INTERNAL_FAILURE; + case AWS_ERROR_S3_SLOW_DOWN: + return Aws::Client::CoreErrors::SLOW_DOWN; + case AWS_ERROR_S3_INVALID_RESPONSE_STATUS: + case AWS_ERROR_S3_RESPONSE_CHECKSUM_MISMATCH: + case AWS_ERROR_S3_CHECKSUM_CALCULATION_FAILED: + case AWS_ERROR_S3_LIST_PARTS_PARSE_FAILED: + case AWS_ERROR_S3_RESUMED_PART_CHECKSUM_MISMATCH: + case AWS_ERROR_S3_FILE_MODIFIED: + case AWS_ERROR_S3_INTERNAL_PART_SIZE_MISMATCH_RETRYING_WITH_RANGE: + case AWS_ERROR_S3_RECV_FILE_ALREADY_EXISTS: + case AWS_ERROR_S3_RECV_FILE_NOT_FOUND: + return Aws::Client::CoreErrors::VALIDATION; + case AWS_ERROR_S3_CANCELED: + return Aws::Client::CoreErrors::USER_CANCELLED; + case AWS_ERROR_S3_REQUEST_TIME_TOO_SKEWED: + return Aws::Client::CoreErrors::REQUEST_TIME_TOO_SKEWED; + case AWS_ERROR_S3EXPRESS_CREATE_SESSION_FAILED: + return Aws::Client::CoreErrors::CLIENT_SIGNING_FAILURE; + case AWS_ERROR_S3_REQUEST_TIMEOUT: + return Aws::Client::CoreErrors::REQUEST_TIMEOUT; + default: + return Aws::Client::CoreErrors::INTERNAL_FAILURE; + } } -DownloadHandle CrtOperations::DispatchDownload(S3TransferManagerImpl&, const DownloadRequest&) { - return DownloadHandle(); +Aws::Client::AWSError MapCrtError(const Aws::Crt::S3::S3MetaRequestResult& result) { + auto httpRequest = Aws::MakeShared( + CRT_OPERATIONS_LOG_TAG, Aws::Http::URI("/"), Aws::Http::HttpMethod::HTTP_GET); + // StandardHttpResponse's constructor eagerly invokes the request's response-stream factory to + // materialize its body stream; without this the factory is empty and aborts under -fno-exceptions. + httpRequest->SetResponseStreamFactory(Aws::Utils::Stream::DefaultResponseStreamFactoryMethod); + std::shared_ptr response = + Aws::MakeShared(CRT_OPERATIONS_LOG_TAG, httpRequest); + + if (result.errorCode != AWS_ERROR_SUCCESS && result.responseStatus == 0) { + response->SetClientErrorType(Aws::Client::CoreErrors::NETWORK_CONNECTION); + Aws::StringStream ss; + ss << "crtCode: " << result.errorCode << ", " << aws_error_name(result.errorCode) << " - " + << aws_error_str(result.errorCode); + response->SetClientErrorMessage(ss.str()); + response->SetResponseCode(Aws::Http::HttpResponseCode::REQUEST_NOT_MADE); + } else { + response->SetResponseCode(static_cast(result.responseStatus)); + } + + // Copy the borrowed error headers/body out of CRT memory (valid only during this callback). + for (const auto& header : result.errorResponseHeaders) { + response->AddHeader(Aws::Utils::StringUtils::FromByteCursor(header.name), + Aws::Utils::StringUtils::FromByteCursor(header.value)); + } + if (result.errorResponseBody.ptr != nullptr && result.errorResponseBody.len > 0) { + response->GetResponseBody().write(reinterpret_cast(result.errorResponseBody.ptr), + static_cast(result.errorResponseBody.len)); + } else if (result.errorCode != AWS_ERROR_SUCCESS) { + // Bodyless CRT-side failure (e.g. response checksum mismatch): supply a typed client error so + // the marshaller produces a precise error instead of "No response body". + Aws::StringStream ss; + ss << aws_error_str(result.errorCode) << " (" << aws_error_lib_name(result.errorCode) << ": " + << aws_error_name(result.errorCode) << ")"; + response->SetClientErrorMessage(ss.str()); + response->SetClientErrorType(MapCrtErrorCode(result.errorCode)); + } + + return Aws::Client::S3ErrorMarshaller().BuildAWSError(response); } +struct ResolvedSigning { + bool hasOverride = false; + bool isS3Express = false; + Aws::String signingRegion; + Aws::String signingName; +}; + +template +bool ResolveEndpointUri(Aws::S3::Endpoint::S3EndpointProvider& provider, const RequestT& s3Request, + const Aws::String& key, Aws::Http::URI& outUri, Aws::String& outError, + ResolvedSigning* outSigning = nullptr) { + auto outcome = provider.ResolveEndpoint(s3Request.GetEndpointContextParams()); + if (!outcome.IsSuccess()) { + outError = outcome.GetError().GetMessage(); + AWS_LOGSTREAM_ERROR(CRT_OPERATIONS_LOG_TAG, "Endpoint resolution failed: " << outError); + return false; + } + Aws::Endpoint::AWSEndpoint endpoint = outcome.GetResultWithOwnership(); + endpoint.AddPathSegments(key); // plural: splits on '/' so prefixed keys map correctly + outUri = endpoint.GetURI(); + + const auto& attributes = endpoint.GetAttributes(); + if (outSigning && attributes) { + if (attributes->authScheme.GetSigningRegion()) { + outSigning->signingRegion = *attributes->authScheme.GetSigningRegion(); + } else if (attributes->authScheme.GetSigningRegionSet()) { + outSigning->signingRegion = *attributes->authScheme.GetSigningRegionSet(); + } + if (attributes->authScheme.GetSigningName()) { + outSigning->signingName = *attributes->authScheme.GetSigningName(); + } + outSigning->isS3Express = attributes->authScheme.GetName() == "S3ExpressSigner"; + outSigning->hasOverride = + outSigning->isS3Express || !outSigning->signingRegion.empty() || !outSigning->signingName.empty(); + } + return true; } + +void ConfigureEndpointSigning(Aws::Crt::S3::S3MetaRequestOptions& options, const ResolvedSigning& signing, + S3TransferManagerImpl& impl) { + if (!signing.hasOverride) { + return; + } + options.SetSigningConfigFromEndpoint( + Aws::Crt::String(impl.GetRegion().c_str()), Aws::Crt::String(signing.signingRegion.c_str()), + Aws::Crt::String(signing.signingName.c_str()), signing.isS3Express, impl.GetCredentialsProvider()); } + +// Build a CRT HTTP message from an S3 model request. Reads request-specific headers rather than +// GetHeaders() (which injects XML/api-version cruft inappropriate for a CRT PUT/GET) and clears the +// body: aws-c-s3 rejects a request with more than one body source, and single-file upload/download +// carry their body via send_filepath / async writes / recv_filepath, never on the message. +template +std::shared_ptr BuildCrtHttpRequest( + const RequestT& s3Request, const Aws::Http::URI& uri, Aws::Http::HttpMethod method, + const std::shared_ptr& body = nullptr, const Aws::String& contentLength = {}, + const Aws::String& contentType = {}) { + auto httpRequest = Aws::Http::CreateHttpRequest(uri, method, Aws::Utils::Stream::DefaultResponseStreamFactoryMethod); + + for (const auto& header : s3Request.GetRequestSpecificHeaders()) { + httpRequest->SetHeaderValue(header.first, header.second); + } + // Content-Type is carried on the streaming base rather than in GetRequestSpecificHeaders(); forward + // it explicitly so a caller-set type is not dropped. + if (!contentType.empty()) { + httpRequest->SetHeaderValue(Aws::Http::CONTENT_TYPE_HEADER, contentType); + } + if (!contentLength.empty()) { + httpRequest->SetContentLength(contentLength); + } + s3Request.AddQueryStringParameters(httpRequest->GetUri()); + + auto crtRequest = httpRequest->ToCrtHttpRequest(); + if (crtRequest) { + crtRequest->SetBody(body); + } + return crtRequest; } + +// Report the bytes remaining from the stream's current get-position to end-of-stream, restoring +// the position before returning. Callers that stream from a pre-seeked position (e.g. seekg(N)) +// need this "remaining bytes" semantics — an absolute length would over-report by the initial +// offset, which fails as AWS_ERROR_S3_INCORRECT_CONTENT_LENGTH on the wire. Matches AWSClient's +// AddContentLengthToRequest which measures (size - begin_position). +// Returns false and leaves the stream usable if the stream cannot report a position. +bool TryMeasureSeekableStream(Aws::IOStream& stream, uint64_t& outLength) { + const std::streampos current = stream.tellg(); + if (current == std::streampos(-1)) { + stream.clear(); + return false; + } + stream.seekg(0, std::ios_base::end); + const std::streampos end = stream.tellg(); + stream.seekg(current); + if (stream.fail() || end == std::streampos(-1) || end < current) { + stream.clear(); + return false; + } + outLength = static_cast(end - current); + return true; } + +// Drives a non-seekable stream upload: reads the customer's stream and pushes each chunk to the CRT +// via S3MetaRequest::Write, waiting for each write to complete before the next (the CRT permits only +// one outstanding write). Stops on EOF, write error, stream failure, or cancellation. +void RunAsyncWriteDriver(std::shared_ptr state, + std::shared_ptr metaRequest, + std::shared_ptr body) { + Aws::Utils::Array buffer(static_cast(DEFAULT_PART_SIZE_BYTES)); + bool eof = false; + while (!eof) { + if (state->canceled.load()) { + return; + } + + body->read(reinterpret_cast(buffer.GetUnderlyingData()), + static_cast(buffer.GetLength())); + const std::streamsize got = body->gcount(); + + if (body->bad()) { + metaRequest->Cancel(); + return; + } + eof = body->eof(); + + Aws::Crt::ByteCursor cursor; + cursor.len = static_cast(got); + cursor.ptr = buffer.GetUnderlyingData(); + + const int errorCode = metaRequest->Write(cursor, eof).get(); + if (errorCode != 0) { + return; + } + } +} + +// Lowercase keys because the generated S3 result deserializers look headers up by lowercase name. +Aws::Http::HeaderValueCollection ToHeaderValueCollection(const Aws::Crt::Vector& headers) { + Aws::Http::HeaderValueCollection out; + for (const auto& header : headers) { + Aws::String name = Aws::Utils::StringUtils::FromByteCursor(header.name); + Aws::String value = Aws::Utils::StringUtils::FromByteCursor(header.value); + out.emplace(Aws::Utils::StringUtils::ToLower(name.c_str()), std::move(value)); + } + return out; +} + +// Returns false for algorithms the CRT enum does not expose (e.g. MD5), so the caller can skip +// configuring a trailer checksum instead of sending a header without a matching trailer. +bool MapChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm sdkAlgorithm, + Aws::Crt::S3::S3ChecksumAlgorithm& crtAlgorithm) { + switch (sdkAlgorithm) { + case Aws::S3::Model::ChecksumAlgorithm::CRC32: + crtAlgorithm = Aws::Crt::S3::S3ChecksumAlgorithm::Crc32; + return true; + case Aws::S3::Model::ChecksumAlgorithm::CRC32C: + crtAlgorithm = Aws::Crt::S3::S3ChecksumAlgorithm::Crc32c; + return true; + case Aws::S3::Model::ChecksumAlgorithm::SHA1: + crtAlgorithm = Aws::Crt::S3::S3ChecksumAlgorithm::Sha1; + return true; + case Aws::S3::Model::ChecksumAlgorithm::SHA256: + crtAlgorithm = Aws::Crt::S3::S3ChecksumAlgorithm::Sha256; + return true; + case Aws::S3::Model::ChecksumAlgorithm::CRC64NVME: + crtAlgorithm = Aws::Crt::S3::S3ChecksumAlgorithm::Crc64Nvme; + return true; + case Aws::S3::Model::ChecksumAlgorithm::SHA512: + crtAlgorithm = Aws::Crt::S3::S3ChecksumAlgorithm::Sha512; + return true; + case Aws::S3::Model::ChecksumAlgorithm::XXHASH64: + crtAlgorithm = Aws::Crt::S3::S3ChecksumAlgorithm::XXHash64; + return true; + case Aws::S3::Model::ChecksumAlgorithm::XXHASH3: + crtAlgorithm = Aws::Crt::S3::S3ChecksumAlgorithm::XXHash3_64; + return true; + case Aws::S3::Model::ChecksumAlgorithm::XXHASH128: + crtAlgorithm = Aws::Crt::S3::S3ChecksumAlgorithm::XXHash3_128; + return true; + default: + return false; + } +} + +// Fires initiated -> failed on every listener and sets the promise. Used by pre-CRT early-return +// paths so listeners see the same lifecycle they would for a CRT-async failure. +template +void NotifyEarlyUploadFailure(std::shared_ptr state, Aws::Client::AWSError error) { + const auto snapshot = + UploadProgressSnapshot(0, state->totalBytes, nullptr, state->totalBytesHasBeenSet); + for (const auto& listener : state->request.GetTransferListeners()) { + if (!listener) continue; + listener->OnTransferInitiated(state->request, snapshot); + listener->OnTransferFailed(state->request, snapshot); + } + state->promise.set_value(UploadOutcome(std::move(error))); +} + +template +void NotifyEarlyDownloadFailure(std::shared_ptr state, Aws::Client::AWSError error) { + const auto snapshot = DownloadProgressSnapshot(0, 0, nullptr, false); + for (const auto& listener : state->request.GetTransferListeners()) { + if (!listener) continue; + listener->OnTransferInitiated(state->request, snapshot); + listener->OnTransferFailed(state->request, snapshot); + } + state->promise.set_value(DownloadOutcome(std::move(error))); +} + +} // namespace + +UploadHandle CrtOperations::DispatchUpload(S3TransferManagerImpl& impl, const UploadRequest& request) { + auto state = Aws::MakeShared(CRT_OPERATIONS_LOG_TAG, request); + + // Body source is file path (CRT reads via send_filepath), seekable stream (CRT reads attached + // body), or non-seekable stream (we push chunks via async writes on a background thread). + const Aws::String& sourceFilePath = request.GetSourceFilePath(); + const std::shared_ptr body = request.GetBody(); + const bool isFileUpload = !sourceFilePath.empty(); + bool bodyInBadState = false; + + if (isFileUpload) { + Aws::IFStream fileSizeStream(sourceFilePath.c_str(), std::ios_base::binary | std::ios_base::ate); + if (fileSizeStream.good()) { + state->totalBytes = static_cast(fileSizeStream.tellg()); + state->totalBytesHasBeenSet = true; + } + } else if (body) { + // Capture the stream's error state before measuring; TryMeasureSeekableStream calls clear(), + // which would mask a pre-existing failure. + bodyInBadState = body->fail(); + if (request.ContentLengthHasBeenSet()) { + state->totalBytes = request.GetContentLength(); + state->totalBytesHasBeenSet = true; + } + uint64_t measured = 0; + if (TryMeasureSeekableStream(*body, measured) && !state->totalBytesHasBeenSet) { + state->totalBytes = measured; + state->totalBytesHasBeenSet = true; + } + } + + auto handleImpl = Aws::MakeUnique(CRT_OPERATIONS_LOG_TAG); + handleImpl->future = state->promise.get_future(); + handleImpl->state = state; + + assert((isFileUpload || body) && "UploadRequest must carry a file path or a stream body"); + + { + const auto& s3Request = request.GetS3Request(); + if (!s3Request.BucketHasBeenSet()) { + NotifyEarlyUploadFailure(state, Aws::Client::AWSError( + Aws::S3::S3Errors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Bucket]", false)); + return UploadHandle(std::move(handleImpl)); + } + if (!s3Request.KeyHasBeenSet()) { + NotifyEarlyUploadFailure(state, Aws::Client::AWSError( + Aws::S3::S3Errors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Key]", false)); + return UploadHandle(std::move(handleImpl)); + } + } + + // Reject an already-failed body up front; reading from it would silently upload truncated data. + if (bodyInBadState) { + NotifyEarlyUploadFailure(state, Aws::Client::AWSError( + Aws::S3::S3Errors::INVALID_PARAMETER_VALUE, "INVALID_PARAMETER_VALUE", "Input stream in bad state", false)); + return UploadHandle(std::move(handleImpl)); + } + + const auto& s3Request = request.GetS3Request(); + Aws::Http::URI uri; + Aws::String endpointError; + ResolvedSigning signing; + if (!ResolveEndpointUri(impl.GetEndpointProvider(), s3Request, s3Request.GetKey(), uri, endpointError, &signing)) { + NotifyEarlyUploadFailure(state, Aws::Client::AWSError( + Aws::Client::CoreErrors::ENDPOINT_RESOLUTION_FAILURE, "ENDPOINT_RESOLUTION_FAILURE", endpointError, false)); + return UploadHandle(std::move(handleImpl)); + } + + // Async-writes is the unknown-length path; aws-c-s3 rejects it alongside a Content-Length header. + const bool useAsyncWrites = !isFileUpload && !state->totalBytesHasBeenSet; + std::shared_ptr crtBody; + if (!isFileUpload && !useAsyncWrites) { + crtBody = Aws::MakeShared(CRT_OPERATIONS_LOG_TAG, body); + } + Aws::String explicitContentLength; + if (request.ContentLengthHasBeenSet()) { + explicitContentLength = Aws::Utils::StringUtils::to_string(request.GetContentLength()); + } else if (!isFileUpload && state->totalBytesHasBeenSet) { + explicitContentLength = Aws::Utils::StringUtils::to_string(state->totalBytes); + } + auto crtRequest = BuildCrtHttpRequest(s3Request, uri, Aws::Http::HttpMethod::HTTP_PUT, crtBody, explicitContentLength, + s3Request.GetContentType()); + + for (const auto& listener : request.GetTransferListeners()) { + if (listener) { + listener->OnTransferInitiated(request, + UploadProgressSnapshot(0, state->totalBytes, nullptr, state->totalBytesHasBeenSet)); + } + } + + Aws::Crt::ScopedResource options; + if (isFileUpload) { + options = Aws::Crt::S3::S3PutObjectMetaRequestOptions::Create(crtRequest, Aws::Crt::String(sourceFilePath.c_str())); + } else if (!useAsyncWrites) { + options = Aws::Crt::S3::S3PutObjectMetaRequestOptions::Create(crtRequest); + } else { + options = Aws::Crt::S3::S3PutObjectMetaRequestOptions::CreateWithAsyncWrites(crtRequest); + } + if (!options) { + // Initiated already fired above; only fire failed here. + const auto snapshot = + UploadProgressSnapshot(0, state->totalBytes, nullptr, state->totalBytesHasBeenSet); + for (const auto& listener : request.GetTransferListeners()) { + if (listener) listener->OnTransferFailed(request, snapshot); + } + state->promise.set_value(UploadOutcome(Aws::Client::AWSError( + Aws::S3::S3Errors::UNKNOWN, "MetaRequestOptionsAllocationFailure", + "Failed to allocate CRT meta request options.", false))); + return UploadHandle(std::move(handleImpl)); + } + + ConfigureEndpointSigning(*options, signing, impl); + + // If the request asks for a checksum, drive it through the CRT trailer path. Skip when the caller + // already supplied a precomputed x-amz-checksum-* header, so we don't ask the CRT to compute a + // conflicting trailer checksum. + const auto& requestHeaders = s3Request.GetHeaders(); + const bool hasPrecomputedChecksum = + std::any_of(requestHeaders.begin(), requestHeaders.end(), [](const Aws::Http::HeaderValuePair& header) { + return header.first.find("x-amz-checksum-") != Aws::String::npos; + }); + if (s3Request.ChecksumAlgorithmHasBeenSet() && !hasPrecomputedChecksum) { + Aws::Crt::S3::S3ChecksumAlgorithm crtAlgorithm; + if (MapChecksumAlgorithm(s3Request.GetChecksumAlgorithm(), crtAlgorithm)) { + Aws::Crt::S3::S3ChecksumConfig checksumConfig; + checksumConfig.SetLocation(Aws::Crt::S3::S3ChecksumLocation::Trailer).SetChecksumAlgorithm(crtAlgorithm); + options->SetChecksumConfig(checksumConfig); + } else { + AWS_LOGSTREAM_ERROR(CRT_OPERATIONS_LOG_TAG, + "Could not map checksum algorithm to a CRT algorithm; upload will proceed without a " + "trailing checksum and S3 may reject it."); + } + } + + options->SetHeadersCallback([state](const Aws::Crt::Vector& headers, int responseStatus) -> int { + state->responseHeaders = ToHeaderValueCollection(headers); + state->responseStatus = responseStatus; + return AWS_OP_SUCCESS; + }); + + options->SetProgressCallback([state](uint64_t bytesTransferred, uint64_t /*contentLength*/) { + const uint64_t soFar = (state->transferredBytes += bytesTransferred); + for (const auto& listener : state->request.GetTransferListeners()) { + if (listener) { + listener->OnBytesTransferred( + state->request, UploadProgressSnapshot(soFar, state->totalBytes, nullptr, state->totalBytesHasBeenSet)); + } + } + }); + + options->SetFinishCallback([state](const Aws::Crt::S3::S3MetaRequestResult& result) { + if (result.errorCode == AWS_ERROR_SUCCESS) { + auto response = Aws::MakeShared(CRT_OPERATIONS_LOG_TAG); + response->SetS3Result(Aws::S3::Model::PutObjectResult( + Aws::AmazonWebServiceResult( + Aws::Utils::Xml::XmlDocument(), state->responseHeaders, + static_cast(state->responseStatus)))); + for (const auto& listener : state->request.GetTransferListeners()) { + if (listener) { + listener->OnTransferComplete(state->request, + UploadProgressSnapshot(state->totalBytes, state->totalBytes, response, true)); + } + } + state->promise.set_value(UploadOutcome(std::move(*response))); + } else { + auto error = MapCrtError(result); + for (const auto& listener : state->request.GetTransferListeners()) { + if (listener) { + listener->OnTransferFailed(state->request, + UploadProgressSnapshot(state->transferredBytes.load(), state->totalBytes, nullptr, + state->totalBytesHasBeenSet)); + } + } + state->promise.set_value(UploadOutcome(std::move(error))); + } + }); + + auto metaRequest = impl.GetCrtClient().MakeMetaRequest(*options); + if (!metaRequest) { + // Initiated already fired above; only fire failed here. + const auto snapshot = + UploadProgressSnapshot(0, state->totalBytes, nullptr, state->totalBytesHasBeenSet); + for (const auto& listener : request.GetTransferListeners()) { + if (listener) listener->OnTransferFailed(request, snapshot); + } + state->promise.set_value(UploadOutcome(Aws::Client::AWSError( + Aws::S3::S3Errors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false))); + return UploadHandle(std::move(handleImpl)); + } + // Publish under the lock; if Cancel() already raced ahead, this cancels immediately. + state->PublishMetaRequest(metaRequest); + + if (useAsyncWrites) { + impl.GetExecutor().Submit([state, metaRequest, body]() { RunAsyncWriteDriver(state, metaRequest, body); }); + } + + return UploadHandle(std::move(handleImpl)); +} + +DownloadHandle CrtOperations::DispatchDownload(S3TransferManagerImpl& impl, const DownloadRequest& request) { + auto state = Aws::MakeShared(CRT_OPERATIONS_LOG_TAG, request); + + // A DownloadDataReceiver selects the zero-copy stream path: the CRT delivers each part via a body + // callback and the receiver reads it in place, so there is no destination file to write or rename. + const std::shared_ptr& receiver = request.GetDownloadDataReceiver(); + const bool isStreamDownload = receiver != nullptr; + + if (!isStreamDownload) { + state->destinationFilePath = request.GetDestinationFilePath(); + // Temp file is a sibling of the destination (same filesystem) so the final rename is atomic. + state->tempFilePath = + state->destinationFilePath + ".s3tmp." + Aws::String(Aws::Utils::UUID::RandomUUID()); + } + + auto handleImpl = Aws::MakeUnique(CRT_OPERATIONS_LOG_TAG); + handleImpl->future = state->promise.get_future(); + handleImpl->state = state; + + const auto& s3Request = request.GetS3Request(); + + if (!s3Request.BucketHasBeenSet()) { + NotifyEarlyDownloadFailure(state, Aws::Client::AWSError( + Aws::S3::S3Errors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Bucket]", false)); + return DownloadHandle(std::move(handleImpl)); + } + if (!s3Request.KeyHasBeenSet()) { + NotifyEarlyDownloadFailure(state, Aws::Client::AWSError( + Aws::S3::S3Errors::MISSING_PARAMETER, "MISSING_PARAMETER", "Missing required field [Key]", false)); + return DownloadHandle(std::move(handleImpl)); + } + + Aws::Http::URI uri; + Aws::String endpointError; + ResolvedSigning signing; + if (!ResolveEndpointUri(impl.GetEndpointProvider(), s3Request, s3Request.GetKey(), uri, endpointError, &signing)) { + NotifyEarlyDownloadFailure(state, Aws::Client::AWSError( + Aws::Client::CoreErrors::ENDPOINT_RESOLUTION_FAILURE, "ENDPOINT_RESOLUTION_FAILURE", endpointError, false)); + return DownloadHandle(std::move(handleImpl)); + } + + auto crtRequest = BuildCrtHttpRequest(s3Request, uri, Aws::Http::HttpMethod::HTTP_GET); + + for (const auto& listener : request.GetTransferListeners()) { + if (listener) { + listener->OnTransferInitiated(request, DownloadProgressSnapshot(0, 0, nullptr, false)); + } + } + + Aws::Crt::ScopedResource options; + if (isStreamDownload) { + options = Aws::Crt::S3::S3GetObjectMetaRequestOptions::Create( + crtRequest, + Aws::Crt::S3::S3MetaRequestOptions::BodyCallbackEx( + [receiver](Aws::Crt::ByteCursor body, uint64_t rangeStart, Aws::Crt::S3::S3BufferTicket& ticket) -> int { + // The ticket handed to the callback is borrowed; Acquire() an owning reference so the + // buffer survives past the callback. The bytes come from the body cursor argument; + // the ticket only owns lifetime. + receiver->OnDataReceived(S3DownloadBuffer(ticket.Acquire(), body, rangeStart)); + return AWS_OP_SUCCESS; + })); + } else { + options = Aws::Crt::S3::S3GetObjectMetaRequestOptions::Create( + crtRequest, Aws::Crt::String(state->tempFilePath.c_str())); + } + if (!options) { + // Initiated already fired above; only fire failed here. + const auto snapshot = DownloadProgressSnapshot(0, state->totalBytes, nullptr, state->totalBytesHasBeenSet); + for (const auto& listener : request.GetTransferListeners()) { + if (listener) listener->OnTransferFailed(request, snapshot); + } + state->promise.set_value(DownloadOutcome(Aws::Client::AWSError( + Aws::S3::S3Errors::UNKNOWN, "MetaRequestOptionsAllocationFailure", + "Failed to allocate CRT meta request options.", false))); + return DownloadHandle(std::move(handleImpl)); + } + + ConfigureEndpointSigning(*options, signing, impl); + + // location MUST be None for a GET: leaving it Trailer (the S3ChecksumConfig default) causes + // aws-c-s3 to sign the bodyless GET with x-amz-content-sha256: STREAMING-UNSIGNED-PAYLOAD-TRAILER, + // which S3 rejects as invalid. + if (s3Request.ChecksumModeHasBeenSet() && s3Request.GetChecksumMode() == Aws::S3::Model::ChecksumMode::ENABLED) { + Aws::Crt::S3::S3ChecksumConfig checksumConfig; + checksumConfig.SetLocation(Aws::Crt::S3::S3ChecksumLocation::None).SetValidateResponseChecksum(true); + options->SetChecksumConfig(checksumConfig); + } + + options->SetHeadersCallback([state](const Aws::Crt::Vector& headers, int responseStatus) -> int { + state->responseHeaders = ToHeaderValueCollection(headers); + state->responseStatus = responseStatus; + return AWS_OP_SUCCESS; + }); + + options->SetProgressCallback([state](uint64_t bytesTransferred, uint64_t contentLength) { + // The whole-object size is learned here (not known up-front for downloads). + if (!state->totalBytesHasBeenSet && contentLength > 0) { + state->totalBytes = contentLength; + state->totalBytesHasBeenSet = true; + } + const uint64_t soFar = (state->transferredBytes += bytesTransferred); + for (const auto& listener : state->request.GetTransferListeners()) { + if (listener) { + listener->OnBytesTransferred( + state->request, DownloadProgressSnapshot(soFar, state->totalBytes, nullptr, state->totalBytesHasBeenSet)); + } + } + }); + + options->SetFinishCallback([state, isStreamDownload](const Aws::Crt::S3::S3MetaRequestResult& result) { + if (result.errorCode == AWS_ERROR_SUCCESS) { + // File path only: promote the temp file to the customer's destination. On Windows, MoveFileW + // won't overwrite, so remove an existing destination first (small non-atomic window); POSIX + // rename is atomic. + if (!isStreamDownload) { +#ifdef _WIN32 + Aws::FileSystem::RemoveFileIfExists(state->destinationFilePath.c_str()); +#endif + if (!Aws::FileSystem::RelocateFileOrDirectory(state->tempFilePath.c_str(), + state->destinationFilePath.c_str())) { + Aws::FileSystem::RemoveFileIfExists(state->tempFilePath.c_str()); + for (const auto& listener : state->request.GetTransferListeners()) { + if (listener) { + listener->OnTransferFailed(state->request, + DownloadProgressSnapshot(state->transferredBytes.load(), state->totalBytes, + nullptr, state->totalBytesHasBeenSet)); + } + } + state->promise.set_value(DownloadOutcome(Aws::Client::AWSError( + Aws::S3::S3Errors::UNKNOWN, "FileRenameFailure", + "Downloaded data could not be moved to the destination path.", false))); + return; + } + } + + auto response = Aws::MakeShared(CRT_OPERATIONS_LOG_TAG); + response->SetS3Result(Aws::S3::Model::GetObjectResult( + Aws::AmazonWebServiceResult( + Aws::Utils::Stream::ResponseStream(), Aws::Http::HeaderValueCollection(state->responseHeaders), + static_cast(state->responseStatus)))); + for (const auto& listener : state->request.GetTransferListeners()) { + if (listener) { + listener->OnTransferComplete( + state->request, + DownloadProgressSnapshot(state->totalBytes, state->totalBytes, response, state->totalBytesHasBeenSet)); + } + } + state->promise.set_value(DownloadOutcome(std::move(*response))); + } else { + // aws-c-s3 does not delete recv_filepath on failure unless recv_file_delete_on_failure is set; + // we don't set it, so clean up the temp file ourselves. + if (!isStreamDownload) { + Aws::FileSystem::RemoveFileIfExists(state->tempFilePath.c_str()); + } + auto error = MapCrtError(result); + for (const auto& listener : state->request.GetTransferListeners()) { + if (listener) { + listener->OnTransferFailed(state->request, + DownloadProgressSnapshot(state->transferredBytes.load(), state->totalBytes, + nullptr, state->totalBytesHasBeenSet)); + } + } + state->promise.set_value(DownloadOutcome(std::move(error))); + } + }); + + auto metaRequest = impl.GetCrtClient().MakeMetaRequest(*options); + if (!metaRequest) { + // Initiated already fired above; only fire failed here. + const auto snapshot = DownloadProgressSnapshot(0, state->totalBytes, nullptr, state->totalBytesHasBeenSet); + for (const auto& listener : request.GetTransferListeners()) { + if (listener) listener->OnTransferFailed(request, snapshot); + } + state->promise.set_value(DownloadOutcome(Aws::Client::AWSError( + Aws::S3::S3Errors::INTERNAL_FAILURE, "INTERNAL_FAILURE", "Unable to create s3 meta request", false))); + return DownloadHandle(std::move(handleImpl)); + } + state->PublishMetaRequest(metaRequest); + + return DownloadHandle(std::move(handleImpl)); +} + +} // namespace Internal +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/internal/S3TransferManagerImpl.cpp b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/internal/S3TransferManagerImpl.cpp new file mode 100644 index 00000000000..05c0379df29 --- /dev/null +++ b/src/aws-cpp-sdk-s3-transfer/source/s3-transfer/internal/S3TransferManagerImpl.cpp @@ -0,0 +1,68 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include +#include + +namespace Aws { +namespace S3 { +namespace Transfer { + +static const char* const S3_TRANSFER_LOG_TAG = "S3TransferManagerImpl"; + +S3TransferManagerImpl::S3TransferManagerImpl(const S3TransferManagerConfiguration& config) + : m_region(config.region) { + Aws::Crt::Auth::CredentialsProviderChainDefaultConfig providerConfig; + m_credentialsProvider = + Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(providerConfig, + Aws::Crt::ApiAllocator()); + if (!m_credentialsProvider) { + AWS_LOGSTREAM_ERROR(S3_TRANSFER_LOG_TAG, "Failed to create CRT default credentials provider chain."); + return; + } + + // Part size and multipart threshold are set explicitly; the CRT's own defaults differ (dynamic + // part size, threshold == part size). + Aws::Crt::S3::S3ClientConfig clientConfig(m_credentialsProvider); + clientConfig.SetRegion(Aws::Crt::String(m_region.c_str())) + .SetPartSize(config.partSize) + .SetMultipartUploadThreshold(config.multipartUploadThreshold) + .SetThroughputTargetGbps(config.throughputTargetGbps); + if (config.tlsConnectionOptions) { + clientConfig.SetTlsConnectionOptions(*config.tlsConnectionOptions); + } + + m_crtClient = Aws::MakeUnique(S3_TRANSFER_LOG_TAG, clientConfig); + if (!m_crtClient || !*m_crtClient) { + AWS_LOGSTREAM_ERROR(S3_TRANSFER_LOG_TAG, + "Failed to construct CRT S3 client, error: " + << (m_crtClient ? m_crtClient->LastError() : -1)); + m_crtClient = nullptr; + return; + } + + m_endpointProvider = Aws::MakeUnique(S3_TRANSFER_LOG_TAG); + m_endpointProvider->AccessBuiltInParameters().SetStringParameter("Region", m_region); + + // Honor endpointOverride like S3Client / S3CrtClient. Used for VPC endpoints, LocalStack, minio. + if (!config.endpointOverride.empty()) { + m_endpointProvider->OverrideEndpoint(config.endpointOverride); + } + + if (config.executor) { + m_executor = config.executor; + } else { + m_executor = Aws::MakeShared(S3_TRANSFER_LOG_TAG, + DEFAULT_EXECUTOR_POOL_SIZE); + } +} + +S3TransferManagerImpl::~S3TransferManagerImpl() = default; + +} // namespace Transfer +} // namespace S3 +} // namespace Aws diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/CMakeLists.txt b/tests/aws-cpp-sdk-s3-transfer-integration-tests/CMakeLists.txt new file mode 100644 index 00000000000..96a43d8367c --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/CMakeLists.txt @@ -0,0 +1,41 @@ +add_project(aws-cpp-sdk-s3-transfer-integration-tests + "Integration tests for the AWS S3 Transfer Manager 2.0 C++ library" + aws-cpp-sdk-s3-transfer + aws-cpp-sdk-s3 + testing-resources + aws-cpp-sdk-core) + +file(GLOB AWS_S3_TRANSFER_TEST_SRC + "${CMAKE_CURRENT_SOURCE_DIR}/*.cpp" +) + +set(S3_TRANSFER_TEST_APPLICATION_INCLUDES + "${AWS_NATIVE_SDK_ROOT}/src/aws-cpp-sdk-core/include/" + "${AWS_NATIVE_SDK_ROOT}/generated/src/aws-cpp-sdk-s3/include/" + "${AWS_NATIVE_SDK_ROOT}/src/aws-cpp-sdk-s3-transfer/include/" + "${AWS_NATIVE_SDK_ROOT}/tests/testing-resources/include/" +) + +include_directories(${S3_TRANSFER_TEST_APPLICATION_INCLUDES}) + +if(MSVC AND BUILD_SHARED_LIBS) + add_definitions(-DGTEST_LINKED_AS_SHARED_LIBRARY=1) +endif() + +enable_testing() + +if(PLATFORM_ANDROID AND BUILD_SHARED_LIBS) + add_library(${PROJECT_NAME} ${AWS_S3_TRANSFER_TEST_SRC}) +else() + add_executable(${PROJECT_NAME} ${AWS_S3_TRANSFER_TEST_SRC}) +endif() + +set_compiler_flags(${PROJECT_NAME}) +set_compiler_warnings(${PROJECT_NAME}) + +target_link_libraries(${PROJECT_NAME} ${PROJECT_LIBS}) + +if(MSVC AND BUILD_SHARED_LIBS) + set_target_properties(${PROJECT_NAME} PROPERTIES LINK_FLAGS "/DELAYLOAD:aws-cpp-sdk-s3-transfer.dll /DELAYLOAD:aws-cpp-sdk-s3.dll /DELAYLOAD:aws-cpp-sdk-core.dll") + target_link_libraries(${PROJECT_NAME} delayimp.lib) +endif() diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/DownloadTests.cpp b/tests/aws-cpp-sdk-s3-transfer-integration-tests/DownloadTests.cpp new file mode 100644 index 00000000000..ade20834499 --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/DownloadTests.cpp @@ -0,0 +1,300 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + * + * Integration tests for single-object download, derived from the S3 Transfer Manager SEP test cases + * in download-single-object.json. The SEP cases assert wire-level GETs (part GETs / ranged GETs); + * TM 2.0 hands a single GET_OBJECT meta request to the CRT, which parallelizes internally. These + * tests assert the observable SEP "outcomes": success/error, totalBytes, byte-identical content + * written to the destination file, checksum-validation success, progress lifecycle, and cancel. + * + * NOTE: GetObjectResult (and therefore DownloadResponse / DownloadOutcome) is move-only. Never copy + * the outcome; bind it by value from future::get() (which moves) and read GetResult() by reference. + */ +#include "RecordingProgressListener.h" +#include "S3TransferTestFixture.h" + +#include +#include +#include + +using namespace Aws::S3::Transfer; +using namespace S3TransferIntegrationTests; + +namespace { + +class DownloadTests : public S3TransferTestFixture { + protected: + S3TransferManagerConfiguration MakeConfig() { + S3TransferManagerConfiguration config; + config.region = Aws::Region::AWS_TEST_REGION; + return config; + } + + // Put an object of `size` deterministic bytes directly via S3 (test setup, not the code under test). + // Returns the local source path (so the test can compare hashes) and the object key. + Aws::String SeedObject(uint64_t size, const Aws::String& key) { + Aws::String sourcePath = MakeLocalFileOfSize(size, "seed"); + Aws::S3::Model::PutObjectRequest put; + put.SetBucket(s_bucketName); + put.SetKey(key); + put.SetBody(Aws::MakeShared(ALLOCATION_TAG, sourcePath.c_str(), + std::ios_base::in | std::ios_base::binary)); + auto outcome = s_s3Client->PutObject(put); + EXPECT_TRUE(outcome.IsSuccess()) << outcome.GetError().GetMessage(); + return sourcePath; + } + + DownloadRequest MakeDownloadRequest(const Aws::String& key, const Aws::String& destPath, + const std::shared_ptr& listener, + bool checksumEnabled = false) { + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + if (checksumEnabled) { + getRequest.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED); + } + Aws::Vector> listeners; + if (listener) { + listeners.push_back(listener); + } + return DownloadRequest(getRequest, destPath, listeners); + } +}; + +// SEP: "Test download ... single object download (object size < part size)" +TEST_F(DownloadTests, SingleObjectDownloadBelowPartSize) { + const uint64_t size = 1048576; // 1 MiB, matches SEP Content-Length + const Aws::String key = UniqueKey(); + Aws::String sourcePath = SeedObject(size, key); + Aws::String sourceHash = FileSha256(sourcePath); + + Aws::String destPath = LocalTempPath("download-small"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(MakeDownloadRequest(key, destPath, listener)).CompletionFuture().get(); + + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(size, FileSize(destPath)); + EXPECT_EQ(sourceHash, FileSha256(destPath)); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// SEP: "Test download ... multipart download (object size > part size)" +// 24 MiB > 8 MiB part size => the CRT issues parallel ranged GETs internally; we assert the +// reassembled file is byte-identical. +TEST_F(DownloadTests, MultipartDownloadAbovePartSize) { + const uint64_t size = 25165824; // 24 MiB, matches SEP + const Aws::String key = UniqueKey(); + Aws::String sourcePath = SeedObject(size, key); + Aws::String sourceHash = FileSha256(sourcePath); + + Aws::String destPath = LocalTempPath("download-mpu"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(MakeDownloadRequest(key, destPath, listener)).CompletionFuture().get(); + + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(size, FileSize(destPath)); + EXPECT_EQ(sourceHash, FileSha256(destPath)); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// SEP: "Test download ... multipart download with uneven last part" +TEST_F(DownloadTests, MultipartDownloadUnevenLastPart) { + const uint64_t size = 10485760; // 10 MiB with 8 MiB part size => 8 MiB + 2 MiB + const Aws::String key = UniqueKey(); + Aws::String sourcePath = SeedObject(size, key); + Aws::String sourceHash = FileSha256(sourcePath); + + Aws::String destPath = LocalTempPath("download-uneven"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(MakeDownloadRequest(key, destPath, listener)).CompletionFuture().get(); + + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(size, FileSize(destPath)); + EXPECT_EQ(sourceHash, FileSha256(destPath)); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// SEP: "Test download ... checksum validation success" +// ChecksumMode=ENABLED on the underlying GetObjectRequest; the CRT validates per-part checksums. +TEST_F(DownloadTests, DownloadWithChecksumValidationSucceeds) { + const uint64_t size = 1048576; // 1 MiB + const Aws::String key = UniqueKey(); + // Seed with a checksum so the object carries one for validation. + Aws::String sourcePath = MakeLocalFileOfSize(size, "seed-crc"); + Aws::String sourceHash = FileSha256(sourcePath); + Aws::S3::Model::PutObjectRequest put; + put.SetBucket(s_bucketName); + put.SetKey(key); + put.SetChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm::CRC32); + put.SetBody(Aws::MakeShared(ALLOCATION_TAG, sourcePath.c_str(), + std::ios_base::in | std::ios_base::binary)); + ASSERT_TRUE(s_s3Client->PutObject(put).IsSuccess()); + + Aws::String destPath = LocalTempPath("download-crc"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = + manager.Download(MakeDownloadRequest(key, destPath, listener, /*checksumEnabled*/ true)).CompletionFuture().get(); + + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(sourceHash, FileSha256(destPath)); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// SEP: progress lifecycle for downloads (initiated once, monotonic bytes, single complete). +TEST_F(DownloadTests, ProgressLifecycleOrdering) { + const uint64_t size = 10485760; // 10 MiB + const Aws::String key = UniqueKey(); + Aws::String sourcePath = SeedObject(size, key); + + Aws::String destPath = LocalTempPath("download-progress"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(MakeDownloadRequest(key, destPath, listener)).CompletionFuture().get(); + + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(1, listener->initiatedCount.load()); + EXPECT_GE(listener->bytesTransferredCount.load(), 1); + EXPECT_EQ(1, listener->completeCount.load()); + EXPECT_EQ(0, listener->failedCount.load()); + EXPECT_FALSE(listener->sawBytesBeforeInitiated.load()); + EXPECT_FALSE(listener->sawNonMonotonic.load()); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// SEP: "Test download ... error handling" — nonexistent key => failed outcome + transferFailed. +TEST_F(DownloadTests, DownloadNonexistentObjectFails) { + const Aws::String key = UniqueKey(); // never uploaded + Aws::String destPath = LocalTempPath("download-missing"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(MakeDownloadRequest(key, destPath, listener)).CompletionFuture().get(); + + EXPECT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(1, listener->failedCount.load()); + EXPECT_EQ(0, listener->completeCount.load()); + // On failure the destination must not be left as a successful file. + EXPECT_FALSE(ObjectExists(key)); + + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// SEP request/response mapping: DownloadResponse wraps a GetObjectResult and must carry +// the observable response fields (ContentLength, ETag). +TEST_F(DownloadTests, DownloadResponseCarriesContentLengthAndETag) { + const uint64_t size = 1048576; // 1 MiB + const Aws::String key = UniqueKey(); + Aws::String sourcePath = SeedObject(size, key); + + Aws::String destPath = LocalTempPath("download-response"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(MakeDownloadRequest(key, destPath, listener)).CompletionFuture().get(); + + AWS_ASSERT_SUCCESS(outcome); + ASSERT_TRUE(outcome.GetResult().S3ResultHasBeenSet()); + const auto& s3Result = outcome.GetResult().GetS3Result(); + EXPECT_EQ(static_cast(size), s3Result.GetContentLength()); + EXPECT_FALSE(s3Result.GetETag().empty()); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// SEP failure handling for file-based download: if the request fails, the destination file +// must not be left behind. +TEST_F(DownloadTests, DownloadFailureLeavesNoDestinationFile) { + const Aws::String key = UniqueKey(); // never uploaded + Aws::String destPath = LocalTempPath("download-failure-cleanup"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(MakeDownloadRequest(key, destPath, listener)).CompletionFuture().get(); + + EXPECT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(1, listener->failedCount.load()); + EXPECT_FALSE(FileExists(destPath)) << "Destination file must not be left behind on failed download"; +} + +// SEP file write mode: by default, an existing destination file must be overwritten. +TEST_F(DownloadTests, DownloadOverwritesExistingFile) { + const uint64_t size = 1048576; // 1 MiB + const Aws::String key = UniqueKey(); + Aws::String sourcePath = SeedObject(size, key); + Aws::String sourceHash = FileSha256(sourcePath); + + // Pre-create the destination with unrelated content that is a different size. + Aws::String destPath = LocalTempPath("download-overwrite"); + { + Aws::OFStream out(destPath.c_str(), std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + const Aws::String preexisting(4096, 'X'); + out.write(preexisting.data(), static_cast(preexisting.size())); + } + ASSERT_EQ(4096u, FileSize(destPath)); + + auto listener = Aws::MakeShared(ALLOCATION_TAG); + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(MakeDownloadRequest(key, destPath, listener)).CompletionFuture().get(); + + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(size, FileSize(destPath)); + EXPECT_EQ(sourceHash, FileSha256(destPath)); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// Ranged GET — SEP download-single-object case "ranged GET single object". S3Crt equivalent: +// TestObjectOperations at BucketAndObjectOperationTest.cpp:564. bytes=128-1024 inclusive => 897. +TEST_F(DownloadTests, DownloadWithByteRangeReturnsRangeSize) { + const uint64_t objectSize = 1048576; + const Aws::String key = UniqueKey(); + Aws::String sourcePath = MakeLocalFileOfSize(objectSize, "range-seed"); + Aws::S3::Model::PutObjectRequest put; + put.SetBucket(s_bucketName); + put.SetKey(key); + put.SetBody(Aws::MakeShared(ALLOCATION_TAG, sourcePath.c_str(), + std::ios_base::in | std::ios_base::binary)); + AWS_ASSERT_SUCCESS(s_s3Client->PutObject(put)); + + const Aws::String destPath = LocalTempPath("range-dest"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + getRequest.SetRange("bytes=128-1024"); + DownloadRequest request(getRequest, destPath); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(request).CompletionFuture().get(); + + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(897u, FileSize(destPath)); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +} // namespace diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/RecordingProgressListener.h b/tests/aws-cpp-sdk-s3-transfer-integration-tests/RecordingProgressListener.h new file mode 100644 index 00000000000..ce2a9352409 --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/RecordingProgressListener.h @@ -0,0 +1,65 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include + +#include +#include + +namespace S3TransferIntegrationTests { + +/** + * Records the ordering and byte counts of the SEP progress lifecycle so tests can assert that + * OnTransferInitiated fires once before any bytes, OnBytesTransferred is monotonic, and exactly + * one terminal event (complete OR failed) fires. Thread-safe: the CRT invokes these from its own + * event-loop threads. + */ +template +class RecordingProgressListenerT : public ListenerBase { + public: + std::atomic initiatedCount{0}; + std::atomic bytesTransferredCount{0}; + std::atomic completeCount{0}; + std::atomic failedCount{0}; + std::atomic lastTransferredBytes{0}; + std::atomic maxTransferredBytes{0}; + std::atomic sawBytesBeforeInitiated{false}; + std::atomic sawNonMonotonic{false}; + + void OnTransferInitiated(const RequestT&, const SnapshotT& snapshot) override { + initiatedCount++; + (void)snapshot; + } + + void OnBytesTransferred(const RequestT&, const SnapshotT& snapshot) override { + bytesTransferredCount++; + if (initiatedCount.load() == 0) { + sawBytesBeforeInitiated = true; + } + const uint64_t transferred = snapshot.GetTransferredBytes(); + if (transferred < lastTransferredBytes.load()) { + sawNonMonotonic = true; + } + lastTransferredBytes = transferred; + uint64_t prevMax = maxTransferredBytes.load(); + while (transferred > prevMax && !maxTransferredBytes.compare_exchange_weak(prevMax, transferred)) { + } + } + + void OnTransferComplete(const RequestT&, const SnapshotT&) override { completeCount++; } + void OnTransferFailed(const RequestT&, const SnapshotT&) override { failedCount++; } +}; + +using RecordingUploadListener = + RecordingProgressListenerT; + +using RecordingDownloadListener = + RecordingProgressListenerT; + +} // namespace S3TransferIntegrationTests diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/RunTests.cpp b/tests/aws-cpp-sdk-s3-transfer-integration-tests/RunTests.cpp new file mode 100644 index 00000000000..a6f2d5668dd --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/RunTests.cpp @@ -0,0 +1,30 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include +#include +#include + +int main(int argc, char** argv) +{ + Aws::Testing::SetDefaultSigPipeHandler(); + Aws::SDKOptions options; + options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Trace; + AWS_BEGIN_MEMORY_TEST_EX(options, 1024, 128); + + Aws::Testing::InitPlatformTest(options); + Aws::Testing::ParseArgs(argc, argv); + + Aws::InitAPI(options); + ::testing::InitGoogleTest(&argc, argv); + int exitCode = RUN_ALL_TESTS(); + + Aws::ShutdownAPI(options); + AWS_END_MEMORY_TEST_EX; + Aws::Testing::ShutdownPlatformTest(options); + return exitCode; +} diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/S3ExpressTest.cpp b/tests/aws-cpp-sdk-s3-transfer-integration-tests/S3ExpressTest.cpp new file mode 100644 index 00000000000..b7d3ca61744 --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/S3ExpressTest.cpp @@ -0,0 +1,227 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + * + * S3 Express (directory bucket) coverage — mandated by the S3 Transfer Manager SEP: + * "Testing plans for S3 Transfer Manager features MUST include S3 Express to ensure consistent + * behavior between general purpose and directory buckets." + * + * Adapted from tests/aws-cpp-sdk-s3-crt-integration-tests/S3ExpressTest.cpp — S3Crt drives PutObject + * / GetObject / MPU against a directory bucket. Here we drive Upload and Download through TM 2.0 + * to prove parity. Covers SEP upload-single-object case #10 and download-single-object case #12. + * + * Uses its own suite fixture (not S3TransferTestFixture) because directory buckets have a distinct + * naming/creation protocol: `----x-s3` suffix, LocationInfo + BucketInfo config, and + * an availability-zone location. + */ +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +using namespace Aws::S3::Transfer; + +namespace S3TransferIntegrationTests { + +namespace { +constexpr const char* ALLOCATION_TAG = "S3ExpressTM2"; +constexpr const char* S3_EXPRESS_SUFFIX = "--use1-az6--x-s3"; +constexpr const char* S3_EXPRESS_AZ = "use1-az6"; +constexpr const char* S3_EXPRESS_REGION = "us-east-1"; // S3 Express requires us-east-1 (or the AZ's region) +} // namespace + +class S3ExpressTransferTests : public Aws::Testing::AwsCppSdkGTestSuite { + protected: + static std::shared_ptr s_s3Client; + static Aws::String s_directoryBucket; + + static void SetUpTestSuite() { + Aws::Testing::AwsCppSdkGTestSuite::SetUpTestSuite(); + + Aws::S3::S3ClientConfiguration config; + config.region = S3_EXPRESS_REGION; + s_s3Client = Aws::MakeShared(ALLOCATION_TAG, config); + + s_directoryBucket = ComputeDirectoryBucketName(); + Aws::S3::Model::CreateBucketRequest createBucket; + createBucket.SetBucket(s_directoryBucket); + Aws::S3::Model::CreateBucketConfiguration bucketConfig; + bucketConfig.SetLocation(Aws::S3::Model::LocationInfo() + .WithType(Aws::S3::Model::LocationType::AvailabilityZone) + .WithName(S3_EXPRESS_AZ)); + bucketConfig.SetBucket(Aws::S3::Model::BucketInfo() + .WithType(Aws::S3::Model::BucketType::Directory) + .WithDataRedundancy(Aws::S3::Model::DataRedundancy::SingleAvailabilityZone)); + createBucket.SetCreateBucketConfiguration(bucketConfig); + auto outcome = s_s3Client->CreateBucket(createBucket); + // BucketAlreadyOwnedByYou / OperationAborted are benign in a shared test account. + if (!outcome.IsSuccess() && outcome.GetError().GetResponseCode() != Aws::Http::HttpResponseCode::CONFLICT) { + AWS_ASSERT_SUCCESS(outcome); + } + } + + static void TearDownTestSuite() { + if (s_s3Client) { + EmptyBucket(s_directoryBucket); + Aws::S3::Model::DeleteBucketRequest deleteBucket; + deleteBucket.SetBucket(s_directoryBucket); + s_s3Client->DeleteBucket(deleteBucket); + s_s3Client = nullptr; + } + Aws::Testing::AwsCppSdkGTestSuite::TearDownTestSuite(); + } + + static Aws::String ComputeDirectoryBucketName() { + Aws::String uuid = Aws::Utils::StringUtils::ToLower(Aws::String(Aws::Utils::UUID::RandomUUID()).c_str()); + uuid.erase(std::remove(uuid.begin(), uuid.end(), '-'), uuid.end()); + return Aws::Testing::GetAwsResourcePrefix() + Aws::String("tm2exp") + uuid.substr(0, 12) + S3_EXPRESS_SUFFIX; + } + + static void EmptyBucket(const Aws::String& bucketName) { + Aws::S3::Model::ListObjectsV2Request listRequest; + listRequest.SetBucket(bucketName); + auto listOutcome = s_s3Client->ListObjectsV2(listRequest); + if (!listOutcome.IsSuccess()) { + return; + } + for (const auto& object : listOutcome.GetResult().GetContents()) { + Aws::S3::Model::DeleteObjectRequest deleteRequest; + deleteRequest.SetBucket(bucketName); + deleteRequest.SetKey(object.GetKey()); + s_s3Client->DeleteObject(deleteRequest); + } + } + + S3TransferManagerConfiguration MakeConfig() { + S3TransferManagerConfiguration config; + config.region = S3_EXPRESS_REGION; + return config; + } + + static Aws::String UniqueKey() { return Aws::String(Aws::Utils::UUID::RandomUUID()).c_str(); } +}; + +std::shared_ptr S3ExpressTransferTests::s_s3Client = nullptr; +Aws::String S3ExpressTransferTests::s_directoryBucket; + +// SEP upload-single-object #10: single-part upload to an S3 Express directory bucket succeeds. +TEST_F(S3ExpressTransferTests, DirectoryBucketSinglePartUpload) { + const uint64_t size = 5ULL * 1024 * 1024; // 5 MiB, matches SEP case size + const Aws::String key = UniqueKey(); + + auto body = Aws::MakeShared(ALLOCATION_TAG); + const Aws::String chunk(64 * 1024, 'a'); + for (uint64_t written = 0; written < size; written += chunk.size()) { + body->write(chunk.data(), static_cast(std::min(chunk.size(), size - written))); + } + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_directoryBucket); + putRequest.SetKey(key); + UploadRequest request(putRequest, body); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_directoryBucket); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(static_cast(size), headOutcome.GetResult().GetContentLength()); +} + +// SEP download-single-object #12: single-part download from an S3 Express directory bucket succeeds. +TEST_F(S3ExpressTransferTests, DirectoryBucketSinglePartDownload) { + const uint64_t size = 5ULL * 1024 * 1024; + const Aws::String key = UniqueKey(); + + // Seed the object via the plain S3 client. + auto seedBody = Aws::MakeShared(ALLOCATION_TAG); + const Aws::String chunk(64 * 1024, 'b'); + for (uint64_t written = 0; written < size; written += chunk.size()) { + seedBody->write(chunk.data(), static_cast(std::min(chunk.size(), size - written))); + } + Aws::S3::Model::PutObjectRequest put; + put.SetBucket(s_directoryBucket); + put.SetKey(key); + put.SetBody(seedBody); + auto putOutcome = s_s3Client->PutObject(put); + AWS_ASSERT_SUCCESS(putOutcome); + + // Download via TM 2.0 with a DataReceiver to avoid needing on-disk temp files. + class CountingReceiver : public DownloadDataReceiver { + public: + void OnDataReceived(S3DownloadBuffer buffer) override { m_bytes += buffer.GetData().len; } + uint64_t Bytes() const { return m_bytes.load(); } + private: + std::atomic m_bytes{0}; + }; + auto receiver = Aws::MakeShared(ALLOCATION_TAG); + + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_directoryBucket); + getRequest.SetKey(key); + DownloadRequest request(getRequest, receiver); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(size, receiver->Bytes()); +} + +// Multipart upload to a directory bucket exercises the CRT's CreateMultipartUpload / +// UploadPart / CompleteMultipartUpload path through the S3 Express signer. +TEST_F(S3ExpressTransferTests, DirectoryBucketMultipartUpload) { + const uint64_t size = 24ULL * 1024 * 1024; // above the 16 MiB threshold + const Aws::String key = UniqueKey(); + + auto body = Aws::MakeShared(ALLOCATION_TAG); + const Aws::String chunk(64 * 1024, 'c'); + for (uint64_t written = 0; written < size; written += chunk.size()) { + body->write(chunk.data(), static_cast(std::min(chunk.size(), size - written))); + } + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_directoryBucket); + putRequest.SetKey(key); + UploadRequest request(putRequest, body); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_directoryBucket); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(static_cast(size), headOutcome.GetResult().GetContentLength()); +} + +} // namespace S3TransferIntegrationTests diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/S3TransferTestFixture.cpp b/tests/aws-cpp-sdk-s3-transfer-integration-tests/S3TransferTestFixture.cpp new file mode 100644 index 00000000000..e943982b19e --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/S3TransferTestFixture.cpp @@ -0,0 +1,12 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include "S3TransferTestFixture.h" + +namespace S3TransferIntegrationTests { + +std::shared_ptr S3TransferTestFixture::s_s3Client = nullptr; +Aws::String S3TransferTestFixture::s_bucketName; + +} // namespace S3TransferIntegrationTests diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/S3TransferTestFixture.h b/tests/aws-cpp-sdk-s3-transfer-integration-tests/S3TransferTestFixture.h new file mode 100644 index 00000000000..5d1967f54d5 --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/S3TransferTestFixture.h @@ -0,0 +1,292 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +namespace S3TransferIntegrationTests { + +static const char ALLOCATION_TAG[] = "S3TransferIntegrationTest"; +static const char TEST_BUCKET_BASE[] = "s3transfertests"; +static const unsigned WAIT_MAX_RETRIES = 20; + +/** + * Shared base fixture for Transfer Manager 2.0 integration tests. Owns a plain S3Client used only + * for test scaffolding (bucket lifecycle + verifying objects); the code under test (S3TransferManager) + * is constructed per-test. One bucket is created per test suite and torn down at the end. + * + * Inherits AwsCppSdkGTestSuite so Aws::InitAPI/ShutdownAPI and memory-leak checks run per suite. + */ +class S3TransferTestFixture : public Aws::Testing::AwsCppSdkGTestSuite { + protected: + static std::shared_ptr s_s3Client; + static Aws::String s_bucketName; + + static Aws::String ComputeBucketName() { + static const Aws::String suffix = Aws::String(Aws::Utils::UUID::RandomUUID()).c_str(); + Aws::StringStream s; + s << Aws::Testing::GetAwsResourcePrefix() << TEST_BUCKET_BASE << suffix; + return Aws::Utils::StringUtils::ToLower(s.str().c_str()); + } + + static void SetUpTestSuite() { + Aws::Testing::AwsCppSdkGTestSuite::SetUpTestSuite(); + + Aws::S3::S3ClientConfiguration config; + config.region = Aws::Region::AWS_TEST_REGION; + config.connectTimeoutMs = 5000; + config.requestTimeoutMs = 60000; + s_s3Client = Aws::MakeShared( + ALLOCATION_TAG, Aws::MakeShared(ALLOCATION_TAG), nullptr, config); + + s_bucketName = ComputeBucketName(); + Aws::S3::Model::CreateBucketRequest createBucket; + createBucket.SetBucket(s_bucketName); + // us-east-1 does not accept a LocationConstraint; every other region requires one. + if (Aws::String(Aws::Region::AWS_TEST_REGION) != Aws::Region::US_EAST_1) { + Aws::S3::Model::CreateBucketConfiguration bucketConfig; + bucketConfig.SetLocationConstraint( + Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(Aws::Region::AWS_TEST_REGION)); + createBucket.SetCreateBucketConfiguration(bucketConfig); + } + auto outcome = s_s3Client->CreateBucket(createBucket); + AWS_ASSERT_SUCCESS(outcome); + ASSERT_TRUE(WaitForBucketToPropagate(s_bucketName)); + } + + static void TearDownTestSuite() { + if (s_s3Client) { + EmptyBucket(s_bucketName); + Aws::S3::Model::DeleteBucketRequest deleteBucket; + deleteBucket.SetBucket(s_bucketName); + s_s3Client->DeleteBucket(deleteBucket); + s_s3Client = nullptr; + } + Aws::FileSystem::DeepDeleteDirectory(GetTestFilesDirectory().c_str()); + Aws::Testing::AwsCppSdkGTestSuite::TearDownTestSuite(); + } + + static bool WaitForBucketToPropagate(const Aws::String& bucketName) { + for (unsigned i = 0; i < WAIT_MAX_RETRIES; ++i) { + Aws::S3::Model::ListObjectsV2Request request; + request.SetBucket(bucketName); + if (s_s3Client->ListObjectsV2(request).IsSuccess()) { + return true; + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + return false; + } + + static void EmptyBucket(const Aws::String& bucketName) { + Aws::S3::Model::ListObjectsV2Request listRequest; + listRequest.SetBucket(bucketName); + auto listOutcome = s_s3Client->ListObjectsV2(listRequest); + if (!listOutcome.IsSuccess()) { + return; + } + for (const auto& object : listOutcome.GetResult().GetContents()) { + Aws::S3::Model::DeleteObjectRequest deleteRequest; + deleteRequest.SetBucket(bucketName); + deleteRequest.SetKey(object.GetKey()); + s_s3Client->DeleteObject(deleteRequest); + } + } + + // ---- local file helpers ---- + + static Aws::String UniqueKey() { return Aws::String(Aws::Utils::UUID::RandomUUID()).c_str(); } + + // Per-suite local scratch directory (created on demand, deep-deleted in TearDownTestSuite). + static Aws::String GetTestFilesDirectory() { + Aws::String directory = "S3TransferTests"; + Aws::FileSystem::CreateDirectoryIfNotExists(directory.c_str()); + return directory; + } + + static Aws::String LocalTempPath(const Aws::String& tag) { + Aws::String fileName = tag + "-" + Aws::String(Aws::Utils::UUID::RandomUUID()); + return Aws::FileSystem::Join(GetTestFilesDirectory(), fileName); + } + + // Write a file of exactly `size` deterministic bytes; returns the path. + static Aws::String MakeLocalFileOfSize(uint64_t size, const Aws::String& tag) { + Aws::String path = LocalTempPath(tag); + Aws::OFStream out(path.c_str(), std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + const Aws::String chunk(64 * 1024, '\0'); + Aws::String buffer = chunk; + for (size_t i = 0; i < buffer.size(); ++i) { + buffer[i] = static_cast(i % 251); // deterministic, non-trivial pattern + } + uint64_t written = 0; + while (written < size) { + const uint64_t toWrite = std::min(buffer.size(), size - written); + out.write(buffer.data(), static_cast(toWrite)); + written += toWrite; + } + out.close(); + return path; + } + + static uint64_t FileSize(const Aws::String& path) { + Aws::IFStream in(path.c_str(), std::ios_base::binary | std::ios_base::ate); + if (!in.good()) { + return 0; + } + return static_cast(in.tellg()); + } + + // No dedicated exists()/stat() helper in Aws::FileSystem for C++11; the SDK's own pattern is to + // open the path and check the stream. Matches how FileSize probes above. + static bool FileExists(const Aws::String& path) { + Aws::IFStream in(path.c_str(), std::ios_base::in | std::ios_base::binary); + return in.is_open(); + } + + // SHA256 of a file's contents, base64-encoded, for byte-identical round-trip comparison. + // CalculateSHA256 takes an IOStream&, so open the file as an FStream (in|binary qualifies). + static Aws::String FileSha256(const Aws::String& path) { + Aws::FStream in(path.c_str(), std::ios_base::in | std::ios_base::binary); + Aws::Utils::ByteBuffer digest = Aws::Utils::HashingUtils::CalculateSHA256(in); + return Aws::Utils::HashingUtils::Base64Encode(digest); + } + + // RAII wrapper around a local scratch file. Constructor writes `size` deterministic bytes at a + // unique path (via MakeLocalFileOfSize); destructor deletes the file. Adapted from TM 1.0's + // ScopedTestFile pattern in tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp — replaces the + // MakeLocalFileOfSize + RemoveFileIfExists pair with a single stack object per test. + class ScopedTestFile { + public: + ScopedTestFile(uint64_t size, const Aws::String& tag) + : m_path(MakeLocalFileOfSize(size, tag)) {} + + // Same shape as above but creates a path only (no bytes written) — useful as a scratch + // destination for downloads that will populate the file themselves. + explicit ScopedTestFile(const Aws::String& tag) : m_path(LocalTempPath(tag)) {} + + ~ScopedTestFile() { Aws::FileSystem::RemoveFileIfExists(m_path.c_str()); } + + ScopedTestFile(const ScopedTestFile&) = delete; + ScopedTestFile& operator=(const ScopedTestFile&) = delete; + + const Aws::String& Path() const { return m_path; } + operator const Aws::String&() const { return m_path; } + + private: + Aws::String m_path; + }; + + static bool ObjectExists(const Aws::String& key) { + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + return s_s3Client->HeadObject(head).IsSuccess(); + } + + // Byte-identical file comparator via SHA256. Adapted from TM 1.0's AreFilesSame helper + // (tests/aws-cpp-sdk-transfer-tests/TransferTests.cpp). + static bool AreFilesSame(const Aws::String& lhs, const Aws::String& rhs) { + if (!FileExists(lhs) || !FileExists(rhs)) { + return false; + } + if (FileSize(lhs) != FileSize(rhs)) { + return false; + } + return FileSha256(lhs) == FileSha256(rhs); + } + + // Poll HeadObject until the object is visible (or timeout). Mirrors TM 1.0's + // WaitForObjectToPropagate; supports SSE-C by taking an optional configurator that stamps the + // customer-provided-key headers onto the HeadObject request. + static bool WaitForObjectToPropagate( + const Aws::String& key, + const std::function& configureRequest = nullptr) { + for (unsigned i = 0; i < WAIT_MAX_RETRIES; ++i) { + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + if (configureRequest) { + configureRequest(head); + } + if (s_s3Client->HeadObject(head).IsSuccess()) { + return true; + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + return false; + } + + // Returns the number of in-progress multipart uploads for `key` in the shared test bucket. + // Used by cancel tests to assert the CRT aborts the MPU on the server side when cancelled. + static size_t CountInProgressMultipartUploads(const Aws::String& key) { + Aws::S3::Model::ListMultipartUploadsRequest listRequest; + listRequest.SetBucket(s_bucketName); + listRequest.SetPrefix(key); + auto outcome = s_s3Client->ListMultipartUploads(listRequest); + if (!outcome.IsSuccess()) { + return 0; + } + size_t count = 0; + for (const auto& upload : outcome.GetResult().GetUploads()) { + if (upload.GetKey() == key) { + ++count; + } + } + return count; + } + + // Best-effort cleanup: abort any lingering multipart uploads under `key` so the bucket teardown + // can succeed even if a test intentionally left an MPU pending (e.g. after a failed abort). + static void AbortAnyInProgressMultipartUploads(const Aws::String& key) { + Aws::S3::Model::ListMultipartUploadsRequest listRequest; + listRequest.SetBucket(s_bucketName); + listRequest.SetPrefix(key); + auto outcome = s_s3Client->ListMultipartUploads(listRequest); + if (!outcome.IsSuccess()) { + return; + } + for (const auto& upload : outcome.GetResult().GetUploads()) { + if (upload.GetKey() != key) { + continue; + } + Aws::S3::Model::AbortMultipartUploadRequest abort; + abort.SetBucket(s_bucketName); + abort.SetKey(upload.GetKey()); + abort.SetUploadId(upload.GetUploadId()); + s_s3Client->AbortMultipartUpload(abort); + } + } +}; + +} // namespace S3TransferIntegrationTests diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/StreamTests.cpp b/tests/aws-cpp-sdk-s3-transfer-integration-tests/StreamTests.cpp new file mode 100644 index 00000000000..4188e3f16c7 --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/StreamTests.cpp @@ -0,0 +1,513 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + * + * Stream body upload + stream (DataReceiver) download coverage. Exercises the TM 2.0 API that + * accepts a std::shared_ptr in place of a file path (upload) and a + * std::shared_ptr in place of a destination file (download): + * - Seekable stream upload below/above the multipart threshold + non-seekable with explicit length. + * - Stream body pre-positioned via seekg / partially consumed by read — verify only the remainder + * is uploaded. + * - DataReceiver zero-copy download (single-part + multipart delivery, ordering + integrity). + * - Large stream (32 MiB) MPU round trip. + */ +#include "RecordingProgressListener.h" +#include "S3TransferTestFixture.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace Aws::S3::Transfer; +using namespace S3TransferIntegrationTests; + +namespace { + +// stringstream subclass whose seekoff/seekpos always fail — used to exercise the non-seekable +// upload path (customer must SetContentLength up front). +class NonSeekableStreamBuf : public std::stringbuf { + public: + explicit NonSeekableStreamBuf(const std::string& data) : std::stringbuf(data, std::ios_base::in) {} + + protected: + pos_type seekoff(off_type, std::ios_base::seekdir, std::ios_base::openmode) override { + return pos_type(off_type(-1)); + } + pos_type seekpos(pos_type, std::ios_base::openmode) override { return pos_type(off_type(-1)); } +}; + +class NonSeekableIOStream : public Aws::IOStream { + public: + explicit NonSeekableIOStream(const std::string& data) : Aws::IOStream(&m_buf), m_buf(data) {} + + private: + NonSeekableStreamBuf m_buf; +}; + +// Test receiver that accumulates delivered parts in call order (thread-safe; CRT may deliver +// concurrently). Used by the DataReceiver download tests. +class RecordingDataReceiver : public DownloadDataReceiver { + public: + struct DeliveredPart { + uint64_t rangeStart; + Aws::String bytes; + }; + + void OnDataReceived(S3DownloadBuffer buffer) override { + Aws::Crt::ByteCursor cursor = buffer.GetData(); + Aws::String copy(reinterpret_cast(cursor.ptr), cursor.len); + std::lock_guard lock(m_mutex); + m_parts.push_back({buffer.GetRangeStart(), std::move(copy)}); + } + + Aws::Vector Snapshot() const { + std::lock_guard lock(m_mutex); + return m_parts; + } + + private: + mutable std::mutex m_mutex; + Aws::Vector m_parts; +}; + +class StreamTests : public S3TransferTestFixture { + protected: + S3TransferManagerConfiguration MakeConfig() { + S3TransferManagerConfiguration config; + config.region = Aws::Region::AWS_TEST_REGION; + return config; + } + + // Deterministic in-memory payload — same byte pattern as MakeLocalFileOfSize (i%251). + static Aws::String MakeDeterministicPayload(uint64_t size) { + Aws::String payload(static_cast(size), '\0'); + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i % 251); + } + return payload; + } + + // Same shape but with an offset fill byte, so different tests use distinguishable bytes. + static Aws::String MakeShiftedPayload(uint64_t size, char fillByte) { + Aws::String out(static_cast(size), '\0'); + for (size_t i = 0; i < out.size(); ++i) { + out[i] = static_cast((static_cast(fillByte) + (i % 251)) & 0xFF); + } + return out; + } + + static Aws::String Sha256OfBuffer(const Aws::String& buffer) { + Aws::StringStream stream; + stream.write(buffer.data(), static_cast(buffer.size())); + return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateSHA256(stream)); + } + + static Aws::String Sha256OfBytes(const char* data, size_t size) { + Aws::StringStream s; + s.write(data, static_cast(size)); + return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateSHA256(s)); + } + + // Reassemble delivered stream-download parts in order → SHA256. + static Aws::String ReassembledSha256(Aws::Vector parts) { + std::sort(parts.begin(), parts.end(), + [](const RecordingDataReceiver::DeliveredPart& a, + const RecordingDataReceiver::DeliveredPart& b) { return a.rangeStart < b.rangeStart; }); + Aws::StringStream stream; + for (const auto& part : parts) { + stream.write(part.bytes.data(), static_cast(part.bytes.size())); + } + return Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateSHA256(stream)); + } + + Aws::String SeedObject(uint64_t size, const Aws::String& key) { + Aws::String sourcePath = MakeLocalFileOfSize(size, "seed-stream"); + Aws::S3::Model::PutObjectRequest put; + put.SetBucket(s_bucketName); + put.SetKey(key); + put.SetBody(Aws::MakeShared(ALLOCATION_TAG, sourcePath.c_str(), + std::ios_base::in | std::ios_base::binary)); + auto outcome = s_s3Client->PutObject(put); + EXPECT_TRUE(outcome.IsSuccess()) << outcome.GetError().GetMessage(); + return sourcePath; + } + + // Upload a stream body pre-positioned at `offset` via seekg; verify the object S3 sees is the + // source's tail slice starting at `offset`. Adapted from TM 1.0 + // TransferManager_MultipartTestWithStreamOffset (TransferTests.cpp:1475). + void RunOffsetUpload(uint64_t totalSize, uint64_t offset) { + ASSERT_LT(offset, totalSize); + const Aws::String key = UniqueKey(); + const Aws::String payload = MakeDeterministicPayload(totalSize); + const uint64_t expectedUploadedBytes = totalSize - offset; + const Aws::String expectedHash = + Sha256OfBytes(payload.data() + offset, static_cast(expectedUploadedBytes)); + + auto body = Aws::MakeShared(ALLOCATION_TAG); + body->write(payload.data(), static_cast(payload.size())); + body->seekg(static_cast(offset), std::ios_base::beg); + ASSERT_EQ(static_cast(offset), body->tellg()); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, body); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(static_cast(expectedUploadedBytes), headOutcome.GetResult().GetContentLength()); + + Aws::String destPath = LocalTempPath("stream-offset-verify"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + getRequest.SetResponseStreamFactory([destPath]() { + return Aws::New(ALLOCATION_TAG, destPath.c_str(), + std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + }); + auto getOutcome = s_s3Client->GetObject(getRequest); + AWS_ASSERT_SUCCESS(getOutcome); + EXPECT_EQ(expectedHash, FileSha256(destPath)); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); + } +}; + +// ============================================================================================ +// Stream body upload +// ============================================================================================ + +// SEP: stream-based upload below the multipart threshold => single PutObject. +TEST_F(StreamTests, SeekableStreamUploadBelowThreshold) { + const uint64_t size = 10485760; // 10 MiB + const Aws::String key = UniqueKey(); + Aws::String payload = MakeDeterministicPayload(size); + Aws::String sourceHash = Sha256OfBuffer(payload); + auto body = Aws::MakeShared(ALLOCATION_TAG); + body->write(payload.data(), static_cast(payload.size())); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + UploadRequest request(putRequest, body, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_TRUE(ObjectExists(key)); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); + + Aws::String downloadPath = LocalTempPath("stream-upload-small-verify"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + getRequest.SetResponseStreamFactory([downloadPath]() { + return Aws::New(ALLOCATION_TAG, downloadPath.c_str(), + std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + }); + auto getOutcome = s_s3Client->GetObject(getRequest); + AWS_ASSERT_SUCCESS(getOutcome); + EXPECT_EQ(sourceHash, FileSha256(downloadPath)); + Aws::FileSystem::RemoveFileIfExists(downloadPath.c_str()); +} + +// SEP: stream-based upload above the multipart threshold => CRT runs MPU internally. +TEST_F(StreamTests, SeekableStreamUploadAboveThreshold) { + const uint64_t size = 25165824; // 24 MiB + const Aws::String key = UniqueKey(); + Aws::String payload = MakeDeterministicPayload(size); + Aws::String sourceHash = Sha256OfBuffer(payload); + auto body = Aws::MakeShared(ALLOCATION_TAG); + body->write(payload.data(), static_cast(payload.size())); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + UploadRequest request(putRequest, body, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_TRUE(ObjectExists(key)); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); + + Aws::String downloadPath = LocalTempPath("stream-upload-mpu-verify"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + getRequest.SetResponseStreamFactory([downloadPath]() { + return Aws::New(ALLOCATION_TAG, downloadPath.c_str(), + std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + }); + auto getOutcome = s_s3Client->GetObject(getRequest); + AWS_ASSERT_SUCCESS(getOutcome); + EXPECT_EQ(sourceHash, FileSha256(downloadPath)); + Aws::FileSystem::RemoveFileIfExists(downloadPath.c_str()); +} + +// Non-seekable stream: caller must announce content-length via UploadRequest::SetContentLength. +TEST_F(StreamTests, NonSeekableStreamUploadWithExplicitContentLength) { + const uint64_t size = 1048576; + const Aws::String key = UniqueKey(); + Aws::String body(size, '\0'); + for (uint64_t i = 0; i < size; ++i) { + body[static_cast(i)] = static_cast(i % 251); + } + auto stream = Aws::MakeShared(ALLOCATION_TAG, std::string(body.c_str(), body.size())); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + UploadRequest request(putRequest, stream, {listener}); + request.SetContentLength(size); + ASSERT_TRUE(request.ContentLengthHasBeenSet()); + ASSERT_EQ(size, request.GetContentLength()); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_TRUE(ObjectExists(key)); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(static_cast(size), headOutcome.GetResult().GetContentLength()); +} + +// ============================================================================================ +// Stream offset / partial-read +// ============================================================================================ + +TEST_F(StreamTests, StreamOffsetSingleObjectUpload) { + RunOffsetUpload(/*totalSize*/ 10ULL * 1024 * 1024, /*offset*/ 8); +} + +TEST_F(StreamTests, StreamOffsetMultipartUpload) { + RunOffsetUpload(/*totalSize*/ 24ULL * 1024 * 1024, /*offset*/ 1024); +} + +// Partial read (extraction) before Upload — TM 2.0 must upload only the unread suffix. +TEST_F(StreamTests, PartiallyReadStreamUploadsOnlyRemainder) { + const uint64_t size = 4ULL * 1024 * 1024; + const size_t prefixToConsume = 128; + const Aws::String key = UniqueKey(); + const Aws::String payload = MakeShiftedPayload(size, 'B'); + const Aws::String expectedHash = + Sha256OfBytes(payload.data() + prefixToConsume, payload.size() - prefixToConsume); + + auto body = Aws::MakeShared(ALLOCATION_TAG); + body->write(payload.data(), static_cast(payload.size())); + + Aws::String scratch(prefixToConsume, '\0'); + body->read(&scratch[0], static_cast(prefixToConsume)); + ASSERT_EQ(static_cast(prefixToConsume), body->gcount()); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, body); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(static_cast(size - prefixToConsume), headOutcome.GetResult().GetContentLength()); + + const Aws::String destPath = LocalTempPath("adv-stream-partial"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + getRequest.SetResponseStreamFactory([destPath]() { + return Aws::New(ALLOCATION_TAG, destPath.c_str(), + std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + }); + auto getOutcome = s_s3Client->GetObject(getRequest); + AWS_ASSERT_SUCCESS(getOutcome); + EXPECT_EQ(expectedHash, FileSha256(destPath)); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// Large-stream MPU round trip (32 MiB) on the seekable stream path. +TEST_F(StreamTests, LargeSeekableStreamMultipartRoundTrip) { + const uint64_t size = 32ULL * 1024 * 1024; + const Aws::String key = UniqueKey(); + const Aws::String payload = MakeShiftedPayload(size, 'A'); + const Aws::String sourceHash = Sha256OfBytes(payload.data(), payload.size()); + + auto body = Aws::MakeShared(ALLOCATION_TAG); + body->write(payload.data(), static_cast(payload.size())); + + auto listener = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, body, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(static_cast(size), headOutcome.GetResult().GetContentLength()); + + const Aws::String destPath = LocalTempPath("adv-stream-mpu"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + getRequest.SetResponseStreamFactory([destPath]() { + return Aws::New(ALLOCATION_TAG, destPath.c_str(), + std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + }); + auto getOutcome = s_s3Client->GetObject(getRequest); + AWS_ASSERT_SUCCESS(getOutcome); + EXPECT_EQ(sourceHash, FileSha256(destPath)); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// ============================================================================================ +// DataReceiver (zero-copy) download +// ============================================================================================ + +// Small object: fits in one CRT part, receiver sees a single delivery. +TEST_F(StreamTests, DataReceiverDownloadSingle) { + const uint64_t size = 1048576; + const Aws::String key = UniqueKey(); + Aws::String sourcePath = SeedObject(size, key); + Aws::String sourceHash = FileSha256(sourcePath); + + auto receiver = Aws::MakeShared(ALLOCATION_TAG); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + DownloadRequest request(getRequest, std::static_pointer_cast(receiver), {listener}); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + auto parts = receiver->Snapshot(); + ASSERT_FALSE(parts.empty()); + uint64_t totalBytes = 0; + for (const auto& part : parts) totalBytes += part.bytes.size(); + EXPECT_EQ(size, totalBytes); + EXPECT_EQ(sourceHash, ReassembledSha256(parts)); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +// Multipart-sized object: CRT splits into several parts delivered in object order. +TEST_F(StreamTests, DataReceiverDownloadMultipart) { + const uint64_t size = 25165824; + const Aws::String key = UniqueKey(); + Aws::String sourcePath = SeedObject(size, key); + Aws::String sourceHash = FileSha256(sourcePath); + + auto receiver = Aws::MakeShared(ALLOCATION_TAG); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + DownloadRequest request(getRequest, std::static_pointer_cast(receiver), {listener}); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + auto parts = receiver->Snapshot(); + ASSERT_GE(parts.size(), 2u) << "Expected multipart delivery for a 24 MiB object"; + + uint64_t totalBytes = 0; + for (const auto& part : parts) totalBytes += part.bytes.size(); + EXPECT_EQ(size, totalBytes); + EXPECT_EQ(sourceHash, ReassembledSha256(parts)); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +// A receiver that reassembles parts into a heap buffer indexed by rangeStart, then SHA256s the +// result. Confirms zero-copy delivery is byte-complete regardless of thread-scheduling order. +TEST_F(StreamTests, DataReceiverReassemblesMultipartInOrder) { + const uint64_t size = 20ULL * 1024 * 1024; + const Aws::String key = UniqueKey(); + const Aws::String payload = MakeShiftedPayload(size, 'C'); + const Aws::String sourceHash = Sha256OfBytes(payload.data(), payload.size()); + + auto seedBody = Aws::MakeShared(ALLOCATION_TAG); + seedBody->write(payload.data(), static_cast(payload.size())); + Aws::S3::Model::PutObjectRequest seedPut; + seedPut.SetBucket(s_bucketName); + seedPut.SetKey(key); + seedPut.SetBody(seedBody); + AWS_ASSERT_SUCCESS(s_s3Client->PutObject(seedPut)); + + class ReassemblingReceiver : public DownloadDataReceiver { + public: + explicit ReassemblingReceiver(uint64_t totalSize) : m_buffer(static_cast(totalSize), '\0') {} + void OnDataReceived(S3DownloadBuffer buffer) override { + std::lock_guard lock(m_mutex); + const uint64_t offset = buffer.GetRangeStart(); + const Aws::Crt::ByteCursor cursor = buffer.GetData(); + if (offset + cursor.len > m_buffer.size()) { + m_overflow = true; + return; + } + std::memcpy(&m_buffer[static_cast(offset)], cursor.ptr, cursor.len); + m_bytesReceived += cursor.len; + } + const Aws::String& Buffer() const { return m_buffer; } + uint64_t BytesReceived() const { return m_bytesReceived; } + bool Overflow() const { return m_overflow; } + + private: + std::mutex m_mutex; + Aws::String m_buffer; + std::atomic m_bytesReceived{0}; + std::atomic m_overflow{false}; + }; + auto receiver = Aws::MakeShared(ALLOCATION_TAG, size); + + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + DownloadRequest request(getRequest, receiver); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_FALSE(receiver->Overflow()); + EXPECT_EQ(size, receiver->BytesReceived()); + EXPECT_EQ(sourceHash, Sha256OfBytes(receiver->Buffer().data(), receiver->Buffer().size())); +} + +} // namespace diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/TransferManagerTests.cpp b/tests/aws-cpp-sdk-s3-transfer-integration-tests/TransferManagerTests.cpp new file mode 100644 index 00000000000..ccbe213c84f --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/TransferManagerTests.cpp @@ -0,0 +1,415 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + * + * Cross-cutting S3TransferManager behavior: cancel semantics (including server-side orphan MPU + * cleanup), concurrent transfers on a shared manager, SSE-KMS round trip, and error surface + * (parsed error types, RequestId presence, IfNoneMatch → 304). + */ +#include "RecordingProgressListener.h" +#include "S3TransferTestFixture.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +using namespace Aws::S3::Transfer; +using namespace S3TransferIntegrationTests; + +namespace { + +class TransferManagerTests : public S3TransferTestFixture { + protected: + S3TransferManagerConfiguration MakeConfig() { + S3TransferManagerConfiguration config; + config.region = Aws::Region::AWS_TEST_REGION; + return config; + } +}; + +// -------- from CancelTests.cpp -------- +TEST_F(TransferManagerTests, UploadCancelViaHandle) { + const uint64_t size = 100 * 1024 * 1024; // 100 MiB — large enough that cancel typically wins + const Aws::String key = UniqueKey(); + Aws::String sourcePath = MakeLocalFileOfSize(size, "cancel-upload"); + + auto listener = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, sourcePath, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadHandle handle = manager.Upload(request); + handle.Cancel(); + UploadOutcome outcome = handle.CompletionFuture().get(); + + if (!outcome.IsSuccess()) { + EXPECT_EQ(1, listener->failedCount.load()); + EXPECT_EQ(0, listener->completeCount.load()); + } else { + // The transfer beat the cancel — lifecycle must still be coherent. + EXPECT_EQ(1, listener->completeCount.load()); + EXPECT_EQ(0, listener->failedCount.load()); + } + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +TEST_F(TransferManagerTests, DownloadCancelViaHandle) { + const uint64_t size = 100 * 1024 * 1024; // 100 MiB + const Aws::String key = UniqueKey(); + + // Seed the object first so the cancel-during-download path is exercised. + Aws::String sourcePath = MakeLocalFileOfSize(size, "cancel-seed"); + Aws::S3::Model::PutObjectRequest put; + put.SetBucket(s_bucketName); + put.SetKey(key); + put.SetBody(Aws::MakeShared(ALLOCATION_TAG, sourcePath.c_str(), + std::ios_base::in | std::ios_base::binary)); + ASSERT_TRUE(s_s3Client->PutObject(put).IsSuccess()); + + Aws::String destPath = LocalTempPath("cancel-download"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + DownloadRequest request(getRequest, destPath, {listener}); + + S3TransferManager manager(MakeConfig()); + DownloadHandle handle = manager.Download(request); + handle.Cancel(); + DownloadOutcome outcome = handle.CompletionFuture().get(); + + if (!outcome.IsSuccess()) { + EXPECT_EQ(1, listener->failedCount.load()); + EXPECT_EQ(0, listener->completeCount.load()); + // Per SEP: destination file must not be left behind on failed download. + EXPECT_FALSE(FileExists(destPath)); + } else { + EXPECT_EQ(1, listener->completeCount.load()); + EXPECT_EQ(0, listener->failedCount.load()); + } + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// -------- from CancelOrphanMpuTests.cpp -------- +TEST_F(TransferManagerTests, CancelAbortsServerSideMultipartUpload) { + const uint64_t size = 200 * 1024 * 1024; // 200 MiB + const Aws::String key = UniqueKey(); + Aws::String sourcePath = MakeLocalFileOfSize(size, "cancel-orphan"); + + auto listener = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, sourcePath, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadHandle handle = manager.Upload(request); + + // Wait until the CRT has actually initiated the transfer before cancelling; on a fast machine an + // immediate cancel can arrive before the CRT even opens the file, which the CRT then reports as + // a synchronous success with no MPU. Waiting for `initiatedCount>=1` (or a short timeout) lines + // us up to hit the MPU path. + for (int i = 0; i < 50 && listener->initiatedCount.load() == 0; ++i) { + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + } + handle.Cancel(); + UploadOutcome outcome = handle.CompletionFuture().get(); + + // If the transfer beat the cancel, the object should exist and there's nothing to leak. That's + // a coherent success and not what this test targets — skip the orphan assertion. + if (outcome.IsSuccess()) { + EXPECT_TRUE(ObjectExists(key)); + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + return; + } + + // The CRT aborts the MPU server-side asynchronously; give it a few seconds to converge. + size_t leaked = 0; + for (unsigned i = 0; i < 30; ++i) { + leaked = CountInProgressMultipartUploads(key); + if (leaked == 0) { + break; + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + EXPECT_EQ(0u, leaked) << "cancelled upload left " << leaked << " orphan multipart upload(s) on the server"; + + // Best-effort teardown so bucket teardown succeeds even if the assertion above failed. + AbortAnyInProgressMultipartUploads(key); + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +// -------- from ConcurrencyTests.cpp -------- +TEST_F(TransferManagerTests, ParallelUploadsShareManagerCleanly) { + const size_t kNumTransfers = 8; + // Size is a mix — a couple below the multipart threshold and a couple above — so the manager + // has both single-object and multipart meta-requests in flight at the same time. + const uint64_t sizes[kNumTransfers] = { + 2ULL * 1024 * 1024, + 2ULL * 1024 * 1024, + 6ULL * 1024 * 1024, + 6ULL * 1024 * 1024, + 20ULL * 1024 * 1024, + 20ULL * 1024 * 1024, + 24ULL * 1024 * 1024, + 24ULL * 1024 * 1024, + }; + + S3TransferManager manager(MakeConfig()); + + Aws::Vector sourcePaths(kNumTransfers); + Aws::Vector keys(kNumTransfers); + Aws::Vector> listeners(kNumTransfers); + Aws::Vector handles; + handles.reserve(kNumTransfers); + + // Dispatch all uploads without waiting for any of them — the manager must keep per-transfer + // state cleanly separated across concurrent meta-requests. + for (size_t i = 0; i < kNumTransfers; ++i) { + Aws::StringStream tag; + tag << "concurrent-" << i; + sourcePaths[i] = MakeLocalFileOfSize(sizes[i], tag.str()); + keys[i] = UniqueKey(); + listeners[i] = Aws::MakeShared(ALLOCATION_TAG); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(keys[i]); + UploadRequest request(putRequest, sourcePaths[i], {listeners[i]}); + handles.emplace_back(manager.Upload(request)); + } + + // Collect all outcomes. + Aws::Vector outcomes; + outcomes.reserve(kNumTransfers); + for (auto& handle : handles) { + outcomes.emplace_back(handle.CompletionFuture().get()); + } + + // Per-transfer assertions. + for (size_t i = 0; i < kNumTransfers; ++i) { + ASSERT_TRUE(outcomes[i].IsSuccess()) + << "transfer " << i << " failed: " << outcomes[i].GetError().GetMessage(); + EXPECT_EQ(1, listeners[i]->initiatedCount.load()) << "transfer " << i; + EXPECT_EQ(1, listeners[i]->completeCount.load()) << "transfer " << i; + EXPECT_EQ(0, listeners[i]->failedCount.load()) << "transfer " << i; + EXPECT_FALSE(listeners[i]->sawBytesBeforeInitiated.load()) << "transfer " << i; + EXPECT_FALSE(listeners[i]->sawNonMonotonic.load()) << "transfer " << i; + EXPECT_EQ(sizes[i], listeners[i]->maxTransferredBytes.load()) << "transfer " << i; + + // Verify the object S3 actually stored has the size we asked to upload. + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(keys[i]); + auto headOutcome = s_s3Client->HeadObject(head); + ASSERT_TRUE(headOutcome.IsSuccess()) << "transfer " << i; + EXPECT_EQ(static_cast(sizes[i]), headOutcome.GetResult().GetContentLength()) + << "transfer " << i << " — object content-length must match the source it was uploaded from " + << "(if this fails, the manager is interleaving bodies across transfers)"; + } + + for (const auto& path : sourcePaths) { + Aws::FileSystem::RemoveFileIfExists(path.c_str()); + } +} + +TEST_F(TransferManagerTests, ParallelDownloadsShareManagerCleanly) { + const size_t kNumTransfers = 6; + const uint64_t sizes[kNumTransfers] = { + 1ULL * 1024 * 1024, + 3ULL * 1024 * 1024, + 6ULL * 1024 * 1024, + 10ULL * 1024 * 1024, + 15ULL * 1024 * 1024, + 20ULL * 1024 * 1024, + }; + + // Seed all objects up front via the plain S3 client, one at a time. + Aws::Vector sourcePaths(kNumTransfers); + Aws::Vector sourceHashes(kNumTransfers); + Aws::Vector keys(kNumTransfers); + for (size_t i = 0; i < kNumTransfers; ++i) { + Aws::StringStream tag; + tag << "concurrent-dl-seed-" << i; + sourcePaths[i] = MakeLocalFileOfSize(sizes[i], tag.str()); + sourceHashes[i] = FileSha256(sourcePaths[i]); + keys[i] = UniqueKey(); + Aws::S3::Model::PutObjectRequest put; + put.SetBucket(s_bucketName); + put.SetKey(keys[i]); + put.SetBody(Aws::MakeShared(ALLOCATION_TAG, sourcePaths[i].c_str(), + std::ios_base::in | std::ios_base::binary)); + AWS_ASSERT_SUCCESS(s_s3Client->PutObject(put)); + } + + S3TransferManager manager(MakeConfig()); + + Aws::Vector destPaths(kNumTransfers); + Aws::Vector> listeners(kNumTransfers); + Aws::Vector handles; + handles.reserve(kNumTransfers); + + for (size_t i = 0; i < kNumTransfers; ++i) { + Aws::StringStream tag; + tag << "concurrent-dl-" << i; + destPaths[i] = LocalTempPath(tag.str()); + listeners[i] = Aws::MakeShared(ALLOCATION_TAG); + + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(keys[i]); + DownloadRequest request(getRequest, destPaths[i], {listeners[i]}); + handles.emplace_back(manager.Download(request)); + } + + Aws::Vector outcomes; + outcomes.reserve(kNumTransfers); + for (auto& handle : handles) { + outcomes.emplace_back(handle.CompletionFuture().get()); + } + + for (size_t i = 0; i < kNumTransfers; ++i) { + ASSERT_TRUE(outcomes[i].IsSuccess()) + << "transfer " << i << " failed: " << outcomes[i].GetError().GetMessage(); + EXPECT_EQ(1, listeners[i]->initiatedCount.load()) << "transfer " << i; + EXPECT_EQ(1, listeners[i]->completeCount.load()) << "transfer " << i; + EXPECT_EQ(0, listeners[i]->failedCount.load()) << "transfer " << i; + EXPECT_EQ(sourceHashes[i], FileSha256(destPaths[i])) + << "transfer " << i << " — downloaded body must match its source " + << "(if this fails, the manager is routing bytes to the wrong destination)"; + } + + for (const auto& path : sourcePaths) { + Aws::FileSystem::RemoveFileIfExists(path.c_str()); + } + for (const auto& path : destPaths) { + Aws::FileSystem::RemoveFileIfExists(path.c_str()); + } +} + +// -------- from ServerSideEncryptionTests.cpp -------- +TEST_F(TransferManagerTests, SseKmsRoundTrip) { + const uint64_t size = 1048576; // 1 MiB + const Aws::String key = UniqueKey(); + + Aws::String sourcePath = MakeLocalFileOfSize(size, "sse-kms"); + const Aws::String sourceHash = FileSha256(sourcePath); + + // Upload with SSE-KMS (aws_kms). No KMS key ID means S3 uses the account-default aws/s3 CMK. + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + putRequest.SetServerSideEncryption(Aws::S3::Model::ServerSideEncryption::aws_kms); + UploadRequest uploadRequest(putRequest, sourcePath); + + S3TransferManager manager(MakeConfig()); + UploadOutcome uploadOutcome = manager.Upload(uploadRequest).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(uploadOutcome); + ASSERT_TRUE(uploadOutcome.GetResult().S3ResultHasBeenSet()); + EXPECT_EQ(Aws::S3::Model::ServerSideEncryption::aws_kms, uploadOutcome.GetResult().GetS3Result().GetServerSideEncryption()); + + // HeadObject via the plain S3 client (no SSE headers needed for SSE-KMS — S3 decrypts server-side). + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(Aws::S3::Model::ServerSideEncryption::aws_kms, headOutcome.GetResult().GetServerSideEncryption()); + + // Download. Because SSE-KMS decrypts server-side, no encryption headers are needed on GetObject. + Aws::String destPath = LocalTempPath("sse-kms-dest"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + DownloadRequest downloadRequest(getRequest, destPath); + + DownloadOutcome downloadOutcome = manager.Download(downloadRequest).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(downloadOutcome); + EXPECT_EQ(sourceHash, FileSha256(destPath)); + EXPECT_EQ(Aws::S3::Model::ServerSideEncryption::aws_kms, downloadOutcome.GetResult().GetS3Result().GetServerSideEncryption()); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +// -------- from ErrorTests.cpp -------- +TEST_F(TransferManagerTests, UploadToNonexistentBucketSurfacesParsedError) { + const Aws::String nonexistentBucket = + "tm2-nonexistent-" + Aws::Utils::StringUtils::ToLower(Aws::String(Aws::Utils::UUID::RandomUUID()).c_str()); + auto body = Aws::MakeShared(ALLOCATION_TAG, "Test Object"); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(nonexistentBucket); + putRequest.SetKey("some-key"); + UploadRequest request(putRequest, body); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + + ASSERT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(Aws::S3::S3Errors::NO_SUCH_BUCKET, outcome.GetError().GetErrorType()); + EXPECT_FALSE(outcome.GetError().GetRequestId().empty()); +} + +TEST_F(TransferManagerTests, DownloadOfNonexistentKeySurfacesParsedError) { + const Aws::String key = UniqueKey(); // never uploaded + const Aws::String destPath = LocalTempPath("nosuchkey"); + + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + DownloadRequest request(getRequest, destPath); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(request).CompletionFuture().get(); + + ASSERT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(Aws::S3::S3Errors::NO_SUCH_KEY, outcome.GetError().GetErrorType()); + EXPECT_FALSE(outcome.GetError().GetRequestId().empty()); +} + +TEST_F(TransferManagerTests, DownloadWithIfNoneMatchReturns304) { + const Aws::String key = UniqueKey(); + + // Seed the object and capture its ETag. + auto body = Aws::MakeShared(ALLOCATION_TAG, "Test never modified!"); + Aws::S3::Model::PutObjectRequest put; + put.SetBucket(s_bucketName); + put.SetKey(key); + put.SetContentType("text/plain"); + put.SetBody(body); + auto putOutcome = s_s3Client->PutObject(put); + AWS_ASSERT_SUCCESS(putOutcome); + const Aws::String etag = putOutcome.GetResult().GetETag(); + ASSERT_FALSE(etag.empty()); + + // Conditional GET with the same ETag must produce HTTP 304 (as an error outcome). + const Aws::String destPath = LocalTempPath("not-modified"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + getRequest.SetIfNoneMatch(etag); + DownloadRequest request(getRequest, destPath); + + S3TransferManager manager(MakeConfig()); + DownloadOutcome outcome = manager.Download(request).CompletionFuture().get(); + + ASSERT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(Aws::Http::HttpResponseCode::NOT_MODIFIED, outcome.GetError().GetResponseCode()); +} + + +} // namespace diff --git a/tests/aws-cpp-sdk-s3-transfer-integration-tests/UploadTests.cpp b/tests/aws-cpp-sdk-s3-transfer-integration-tests/UploadTests.cpp new file mode 100644 index 00000000000..6dc79a39fea --- /dev/null +++ b/tests/aws-cpp-sdk-s3-transfer-integration-tests/UploadTests.cpp @@ -0,0 +1,701 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + * + * Upload coverage — SEP upload-single-object cases plus TM 1.0 / S3Crt-derived ports. Covers: + * - Single-object & multipart uploads (size boundaries, uneven last part, byte-identical hash) + * - Progress listener lifecycle + fan-out to multiple listeners + * - Error paths (nonexistent bucket, bad local file, orphan MPU cleanup on failure) + * - Request field pass-through: content-type, user metadata, checksum matrix, composite MPU + * checksum, full-object CRC32, IfNoneMatch, SSE via PutObjectRequest + * - Special/unicode/URI-escape key names, DNS-unfriendly bucket names + * - Empty/zero-byte body, 75 MiB big-file round trip, progress monotonicity + */ +#include "RecordingProgressListener.h" +#include "S3TransferTestFixture.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace Aws::S3::Transfer; +using namespace S3TransferIntegrationTests; +using Aws::Http::HttpResponseCode; +using Aws::S3::Model::ChecksumAlgorithm; +using Aws::S3::Model::PutObjectRequest; +using Aws::Utils::HashingUtils; + +namespace { + +// SEP part size (8 MiB); multipart threshold defaults to 16 MiB. +constexpr uint64_t SEP_PART_SIZE = 8388608; + +// From S3Crt: special-character keys used by TestPutWithSpecialCharactersInKeyName and +// TestKeysWithCrazyCharacterSets. +constexpr const char* URLENCODED_UNICODE_KEY = "TestUnicode%E4%B8%AD%E5%9B%BDKey"; +constexpr const char* URIESCAPE_KEY = "Esc a=pe+Me$"; +constexpr const char* SPECIAL_CHARS_KEY = "foo;jsessionid=40+2"; + +// Table row for the checksum-matrix test, ported from S3Crt +// BucketAndObjectOperationTest.cpp:1526. +struct ChecksumTestCase { + std::function checksumRequestMutator; + HttpResponseCode responseCode; + Aws::String body; +}; + +class UploadTests : public S3TransferTestFixture { + protected: + S3TransferManagerConfiguration MakeConfig() { + S3TransferManagerConfiguration config; + config.region = Aws::Region::AWS_TEST_REGION; + return config; + } + + // Upload a local file of the given size and return the terminal outcome plus the recording listener. + UploadOutcome DoUpload(uint64_t fileSize, const Aws::String& key, + const std::shared_ptr& listener, + Aws::S3::Model::ChecksumAlgorithm checksum = Aws::S3::Model::ChecksumAlgorithm::NOT_SET) { + Aws::String sourcePath = MakeLocalFileOfSize(fileSize, "upload"); + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + if (checksum != Aws::S3::Model::ChecksumAlgorithm::NOT_SET) { + putRequest.SetChecksumAlgorithm(checksum); + } + Aws::Vector> listeners; + if (listener) listeners.push_back(listener); + UploadRequest request(putRequest, sourcePath, listeners); + + S3TransferManager manager(MakeConfig()); + UploadHandle handle = manager.Upload(request); + UploadOutcome outcome = handle.CompletionFuture().get(); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + return outcome; + } + + // Upload a small stream body under the given key and verify HeadObject succeeds. Used by the + // special-character-key tests. + void UploadAndVerifyKey(const Aws::String& key) { + auto body = Aws::MakeShared(ALLOCATION_TAG, "Test Object"); + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + putRequest.SetContentType("text/plain"); + UploadRequest request(putRequest, body); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + ASSERT_TRUE(outcome.IsSuccess()) << outcome.GetError().GetMessage() << " (key=" << key << ")"; + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + ASSERT_TRUE(headOutcome.IsSuccess()) << headOutcome.GetError().GetMessage() << " (key=" << key << ")"; + } + + // Bucket name with a dot in it — breaks virtual-hosted-style addressing; the SDK must fall back + // to URI-segment style. + static Aws::String MakeUnfriendlyBucketName() { + return Aws::Testing::GetAwsResourcePrefix() + "unfriendly." + + Aws::Utils::StringUtils::ToLower(Aws::String(Aws::Utils::UUID::RandomUUID()).c_str()); + } +}; + +// ============================================================================================ +// Basic upload paths (SEP: single-object, multipart, uneven last part) +// ============================================================================================ + +TEST_F(UploadTests, SingleObjectUploadBelowThreshold) { + const uint64_t size = 10485760; + const Aws::String key = UniqueKey(); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + UploadOutcome outcome = DoUpload(size, key, listener); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_TRUE(ObjectExists(key)); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); +} + +TEST_F(UploadTests, MultipartUploadAboveThreshold) { + const uint64_t size = 25165824; + const Aws::String key = UniqueKey(); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + Aws::String sourcePath = MakeLocalFileOfSize(size, "upload-mpu"); + Aws::String sourceHash = FileSha256(sourcePath); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, sourcePath, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_TRUE(ObjectExists(key)); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); + + Aws::String downloadPath = LocalTempPath("upload-mpu-verify"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + getRequest.SetResponseStreamFactory([downloadPath]() { + return Aws::New(ALLOCATION_TAG, downloadPath.c_str(), + std::ios_base::out | std::ios_base::binary | std::ios_base::trunc); + }); + auto getOutcome = s_s3Client->GetObject(getRequest); + AWS_ASSERT_SUCCESS(getOutcome); + EXPECT_EQ(sourceHash, FileSha256(downloadPath)); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(downloadPath.c_str()); +} + +TEST_F(UploadTests, MultipartUploadUnevenLastPart) { + const uint64_t size = SEP_PART_SIZE + 2097152; + const Aws::String key = UniqueKey(); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + UploadOutcome outcome = DoUpload(size, key, listener); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_TRUE(ObjectExists(key)); +} + +TEST_F(UploadTests, SingleObjectUploadWithChecksum) { + const uint64_t size = 10485760; + const Aws::String key = UniqueKey(); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + UploadOutcome outcome = DoUpload(size, key, listener, Aws::S3::Model::ChecksumAlgorithm::CRC32); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_TRUE(ObjectExists(key)); +} + +// ============================================================================================ +// Progress lifecycle & fan-out +// ============================================================================================ + +TEST_F(UploadTests, ProgressLifecycleOrdering) { + const uint64_t size = 10485760; + const Aws::String key = UniqueKey(); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + UploadOutcome outcome = DoUpload(size, key, listener); + AWS_ASSERT_SUCCESS(outcome); + EXPECT_EQ(1, listener->initiatedCount.load()); + EXPECT_GE(listener->bytesTransferredCount.load(), 1); + EXPECT_EQ(1, listener->completeCount.load()); + EXPECT_EQ(0, listener->failedCount.load()); + EXPECT_FALSE(listener->sawBytesBeforeInitiated.load()); + EXPECT_FALSE(listener->sawNonMonotonic.load()); +} + +// Adapted from TM 1.0 TransferManager_MultiPartStreamableByteTest (TransferTests.cpp:1835): assert +// OnBytesTransferred is monotonically non-decreasing and totals the source size on the MPU path. +TEST_F(UploadTests, ProgressMonotonicityOnMultipartUpload) { + const uint64_t size = 25165824; + const Aws::String key = UniqueKey(); + const Aws::String sourcePath = MakeLocalFileOfSize(size, "progress-monotonic"); + + auto listener = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, sourcePath, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + EXPECT_FALSE(listener->sawNonMonotonic.load()); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); + EXPECT_GE(listener->bytesTransferredCount.load(), 1); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +TEST_F(UploadTests, UploadFanOutToMultipleListeners) { + const uint64_t size = 1048576; + const Aws::String key = UniqueKey(); + Aws::String sourcePath = MakeLocalFileOfSize(size, "upload-fanout"); + + auto listenerA = Aws::MakeShared(ALLOCATION_TAG); + auto listenerB = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, sourcePath, {listenerA, listenerB}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + for (const auto& listener : {listenerA, listenerB}) { + EXPECT_EQ(1, listener->initiatedCount.load()); + EXPECT_GE(listener->bytesTransferredCount.load(), 1); + EXPECT_EQ(1, listener->completeCount.load()); + EXPECT_EQ(0, listener->failedCount.load()); + EXPECT_EQ(size, listener->maxTransferredBytes.load()); + } + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +// ============================================================================================ +// Error paths +// ============================================================================================ + +TEST_F(UploadTests, UploadToNonexistentBucketFails) { + const uint64_t size = 1048576; + const Aws::String key = UniqueKey(); + Aws::String sourcePath = MakeLocalFileOfSize(size, "upload-err"); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket("this-bucket-should-not-exist-" + Aws::Utils::StringUtils::ToLower(UniqueKey().c_str())); + putRequest.SetKey(key); + UploadRequest request(putRequest, sourcePath, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + + EXPECT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(1, listener->failedCount.load()); + EXPECT_EQ(0, listener->completeCount.load()); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +// Ported from TM 1.0 BadFileTest — upload from a nonexistent local file surfaces a failed outcome +// and the listener lifecycle (initiated → failed). +TEST_F(UploadTests, UploadOfNonexistentSourceFileFails) { + const Aws::String nonexistentPath = LocalTempPath("does-not-exist"); + const Aws::String key = "UnicornKey"; + + auto listener = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + putRequest.SetContentType("text/plain"); + UploadRequest request(putRequest, nonexistentPath, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + + EXPECT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(1, listener->failedCount.load()); + EXPECT_EQ(0, listener->completeCount.load()); +} + +// SEP upload cases #6/#7 — a failed MPU-sized upload must not leave orphan multipart uploads. We +// use a nonexistent bucket to force real S3 failure during the MPU flow. +TEST_F(UploadTests, MultipartUploadFailureCleansUpOrphanedMpu) { + const uint64_t size = 25165824; + const Aws::String key = UniqueKey(); + const Aws::String nonexistentBucket = + "tm2-nosuch-" + Aws::Utils::StringUtils::ToLower(Aws::String(Aws::Utils::UUID::RandomUUID()).c_str()); + const Aws::String sourcePath = MakeLocalFileOfSize(size, "mpu-error"); + + auto listener = Aws::MakeShared(ALLOCATION_TAG); + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(nonexistentBucket); + putRequest.SetKey(key); + UploadRequest request(putRequest, sourcePath, {listener}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + + EXPECT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(1, listener->failedCount.load()); + EXPECT_EQ(0, listener->completeCount.load()); + EXPECT_FALSE(outcome.GetError().GetMessage().empty()); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +// ============================================================================================ +// Response fields / metadata pass-through +// ============================================================================================ + +TEST_F(UploadTests, UploadResponseCarriesETag) { + const uint64_t size = 1048576; + const Aws::String key = UniqueKey(); + auto listener = Aws::MakeShared(ALLOCATION_TAG); + UploadOutcome outcome = DoUpload(size, key, listener); + AWS_ASSERT_SUCCESS(outcome); + ASSERT_TRUE(outcome.GetResult().S3ResultHasBeenSet()); + EXPECT_FALSE(outcome.GetResult().GetS3Result().GetETag().empty()); +} + +TEST_F(UploadTests, UploadPreservesContentTypeAndUserMetadata) { + const uint64_t size = 1048576; + const Aws::String key = UniqueKey(); + Aws::String sourcePath = MakeLocalFileOfSize(size, "upload-meta"); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + putRequest.SetContentType("application/x-test-type"); + putRequest.AddMetadata("test-key", "test-value"); + UploadRequest request(putRequest, sourcePath, {}); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ("application/x-test-type", headOutcome.GetResult().GetContentType()); + const auto& metadata = headOutcome.GetResult().GetMetadata(); + auto it = metadata.find("test-key"); + ASSERT_NE(metadata.end(), it); + EXPECT_EQ("test-value", it->second); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +// Same as above but on the MPU path — user metadata must land on CreateMultipartUpload. +TEST_F(UploadTests, MultipartUploadPreservesContentTypeAndUserMetadata) { + const uint64_t size = 25165824; + const Aws::String key = UniqueKey(); + Aws::String sourcePath = MakeLocalFileOfSize(size, "mpu-meta"); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + putRequest.SetContentType("application/x-test-mpu"); + putRequest.AddMetadata("test-key-one", "value-one"); + putRequest.AddMetadata("test-key-two", "value-two"); + UploadRequest request(putRequest, sourcePath); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ("application/x-test-mpu", headOutcome.GetResult().GetContentType()); + const auto& metadata = headOutcome.GetResult().GetMetadata(); + auto it1 = metadata.find("test-key-one"); + ASSERT_NE(metadata.end(), it1); + EXPECT_EQ("value-one", it1->second); + auto it2 = metadata.find("test-key-two"); + ASSERT_NE(metadata.end(), it2); + EXPECT_EQ("value-two", it2->second); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +// ============================================================================================ +// Empty / zero-byte body — ported from S3Crt TestNullBody + TestEmptyBody +// ============================================================================================ + +TEST_F(UploadTests, ZeroByteFileUpload) { + const Aws::String key = UniqueKey(); + const Aws::String sourcePath = MakeLocalFileOfSize(0, "empty-file"); + ASSERT_EQ(0u, FileSize(sourcePath)); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, sourcePath); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(0, headOutcome.GetResult().GetContentLength()); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +TEST_F(UploadTests, EmptyStreamUpload) { + const Aws::String key = UniqueKey(); + auto body = Aws::MakeShared(ALLOCATION_TAG, ""); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + UploadRequest request(putRequest, body); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(0, headOutcome.GetResult().GetContentLength()); +} + +// ============================================================================================ +// Special-character keys — ported from S3Crt +// ============================================================================================ + +TEST_F(UploadTests, KeyWithSemicolonAndPlus) { UploadAndVerifyKey(SPECIAL_CHARS_KEY); } + +TEST_F(UploadTests, UnicodeKey) { + const Aws::String unicodeKey = Aws::Utils::StringUtils::URLDecode(URLENCODED_UNICODE_KEY); + UploadAndVerifyKey(unicodeKey); +} + +TEST_F(UploadTests, UriEscapeKey) { UploadAndVerifyKey(URIESCAPE_KEY); } + +// ============================================================================================ +// Virtual-hosted-style addressing — ported from S3Crt TestVirtualAddressingWithUnfriendlyBucketName +// ============================================================================================ + +TEST_F(UploadTests, PutObjectSucceedsOnDnsUnfriendlyBucket) { + const Aws::String bucketName = MakeUnfriendlyBucketName(); + + Aws::S3::Model::CreateBucketRequest createBucket; + createBucket.SetBucket(bucketName); + if (Aws::String(Aws::Region::AWS_TEST_REGION) != Aws::Region::US_EAST_1) { + Aws::S3::Model::CreateBucketConfiguration bucketConfig; + bucketConfig.SetLocationConstraint( + Aws::S3::Model::BucketLocationConstraintMapper::GetBucketLocationConstraintForName(Aws::Region::AWS_TEST_REGION)); + createBucket.SetCreateBucketConfiguration(bucketConfig); + } + auto createOutcome = s_s3Client->CreateBucket(createBucket); + AWS_ASSERT_SUCCESS(createOutcome); + ASSERT_TRUE(WaitForBucketToPropagate(bucketName)); + + auto body = Aws::MakeShared(ALLOCATION_TAG, + "'A program that has not been tested does not work'-- Bjarne Stroustrup"); + const Aws::String key = "WhySoHostile"; + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(bucketName); + putRequest.SetKey(key); + putRequest.SetContentType("text/plain"); + UploadRequest request(putRequest, body); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::DeleteObjectRequest deleteObj; + deleteObj.SetBucket(bucketName); + deleteObj.SetKey(key); + s_s3Client->DeleteObject(deleteObj); + Aws::S3::Model::DeleteBucketRequest deleteBucket; + deleteBucket.SetBucket(bucketName); + s_s3Client->DeleteBucket(deleteBucket); +} + +// ============================================================================================ +// Checksum matrix — 13-row table ported from S3Crt PutObjectChecksum (BucketAndObjectOperationTest.cpp:1526) +// ============================================================================================ + +TEST_F(UploadTests, PutObjectChecksum) { + const Aws::String key = UniqueKey(); + + Aws::Vector testCases{ + // Mismatched checksums → 400 + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32).WithChecksumCRC32("Just runnin' scared each place we go"); + }, + HttpResponseCode::BAD_REQUEST, "Just runnin' scared each place we go"}, + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA1).WithChecksumSHA1("So afraid that he might show"); + }, + HttpResponseCode::BAD_REQUEST, "So afraid that he might show"}, + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA256).WithChecksumSHA256("Yeah, runnin' scared, what would I do"); + }, + HttpResponseCode::BAD_REQUEST, "Yeah, runnin' scared, what would I do"}, + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32C).WithChecksumCRC32C("If he came back and wanted you?"); + }, + HttpResponseCode::BAD_REQUEST, "If he came back and wanted you?"}, + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA512).WithChecksumSHA512("If he came back and wanted you?"); + }, + HttpResponseCode::BAD_REQUEST, "If he came back and wanted you?"}, + {[](PutObjectRequest request) { return request.WithContentMD5("Just runnin' scared, feelin' low"); }, + HttpResponseCode::BAD_REQUEST, "Just runnin' scared, feelin' low"}, + // Correct checksums → OK + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32) + .WithChecksumCRC32(HashingUtils::Base64Encode(HashingUtils::CalculateCRC32("Runnin' scared, you love him so"))); + }, + HttpResponseCode::OK, "Runnin' scared, you love him so"}, + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA1) + .WithChecksumSHA1(HashingUtils::Base64Encode(HashingUtils::CalculateSHA1("Just runnin' scared, afraid to lose"))); + }, + HttpResponseCode::OK, "Just runnin' scared, afraid to lose"}, + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA256) + .WithChecksumSHA256(HashingUtils::Base64Encode(HashingUtils::CalculateSHA256("If he came back, which one would you choose?"))); + }, + HttpResponseCode::OK, "If he came back, which one would you choose?"}, + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::CRC32C) + .WithChecksumCRC32C(HashingUtils::Base64Encode(HashingUtils::CalculateCRC32C("Then all at once he was standing there"))); + }, + HttpResponseCode::OK, "Then all at once he was standing there"}, + {[](PutObjectRequest request) { + return request.WithContentMD5(HashingUtils::Base64Encode(HashingUtils::CalculateMD5("So sure of himself, his head in the air"))); + }, + HttpResponseCode::OK, "So sure of himself, his head in the air"}, + {[](PutObjectRequest request) { + return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA512) + .WithChecksumSHA512(HashingUtils::Base64Encode(HashingUtils::CalculateSHA512("If he came back, which one would you choose?"))); + }, + HttpResponseCode::OK, "If he came back, which one would you choose?"}, + // Algorithm-only → SDK computes → OK + {[](PutObjectRequest request) { return request.WithChecksumAlgorithm(ChecksumAlgorithm::SHA512); }, + HttpResponseCode::OK, "If he came back, which one would you choose?"}, + // MD5 as ChecksumAlgorithm is invalid → 400 + {[](PutObjectRequest request) { return request.WithChecksumAlgorithm(ChecksumAlgorithm::MD5); }, + HttpResponseCode::BAD_REQUEST, "If he came back, which one would you choose?"}, + }; + + for (const auto& testCase : testCases) { + PutObjectRequest putRequest = testCase.checksumRequestMutator(PutObjectRequest().WithBucket(s_bucketName).WithKey(key)); + std::shared_ptr body = + Aws::MakeShared(ALLOCATION_TAG, testCase.body, std::ios_base::in | std::ios_base::binary); + UploadRequest request(putRequest, body); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + + if (testCase.responseCode == HttpResponseCode::OK) { + ASSERT_TRUE(outcome.IsSuccess()) << outcome.GetError().GetMessage() << " (body=" << testCase.body << ")"; + } else { + ASSERT_FALSE(outcome.IsSuccess()) << "expected failure for body=" << testCase.body; + EXPECT_EQ(testCase.responseCode, outcome.GetError().GetResponseCode()) << "body=" << testCase.body; + } + } +} + +// ============================================================================================ +// MPU composite + full-object checksums (SEP upload-single-object #4) +// ============================================================================================ + +TEST_F(UploadTests, MultipartUploadWithCrc32ProducesCompositeChecksum) { + const uint64_t size = 25165824; + const Aws::String key = UniqueKey(); + const Aws::String sourcePath = MakeLocalFileOfSize(size, "composite-crc"); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + putRequest.SetChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm::CRC32); + UploadRequest request(putRequest, sourcePath); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + head.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + const Aws::String checksum = headOutcome.GetResult().GetChecksumCRC32(); + ASSERT_FALSE(checksum.empty()) << "HeadObject did not report a CRC32 checksum"; + EXPECT_NE(Aws::String::npos, checksum.find('-')) << "expected composite \"...-N\" suffix, got: " << checksum; + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +TEST_F(UploadTests, MultipartUploadWithPrecomputedFullObjectCrc32) { + const uint64_t size = 16777216; + const Aws::String key = UniqueKey(); + const Aws::String sourcePath = MakeLocalFileOfSize(size, "full-object-crc"); + + Aws::FStream fileStream(sourcePath.c_str(), std::ios_base::in | std::ios_base::binary); + const Aws::String fullCrc32 = Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateCRC32(fileStream)); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + putRequest.SetChecksumAlgorithm(Aws::S3::Model::ChecksumAlgorithm::CRC32); + putRequest.SetChecksumCRC32(fullCrc32); + UploadRequest request(putRequest, sourcePath); + + S3TransferManager manager(MakeConfig()); + UploadOutcome outcome = manager.Upload(request).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(outcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + head.SetChecksumMode(Aws::S3::Model::ChecksumMode::ENABLED); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + const Aws::String reported = headOutcome.GetResult().GetChecksumCRC32(); + ASSERT_FALSE(reported.empty()) << "HeadObject did not report a CRC32 checksum"; + EXPECT_EQ(Aws::String::npos, reported.find('-')) << "expected full-object checksum, got: " << reported; + EXPECT_EQ(fullCrc32, reported); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); +} + +// ============================================================================================ +// Large-file smoke test — TM 1.0 TransferManager_BigTest at 75 MiB +// ============================================================================================ + +TEST_F(UploadTests, BigFileUploadDownloadRoundTrip) { + const uint64_t size = 75ULL * 1024 * 1024; + const Aws::String key = UniqueKey(); + const Aws::String sourcePath = MakeLocalFileOfSize(size, "big-upload"); + const Aws::String sourceHash = FileSha256(sourcePath); + + Aws::S3::Model::PutObjectRequest putRequest; + putRequest.SetBucket(s_bucketName); + putRequest.SetKey(key); + putRequest.SetContentType("text/plain"); + UploadRequest uploadRequest(putRequest, sourcePath); + + S3TransferManager manager(MakeConfig()); + UploadOutcome uploadOutcome = manager.Upload(uploadRequest).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(uploadOutcome); + + Aws::S3::Model::HeadObjectRequest head; + head.SetBucket(s_bucketName); + head.SetKey(key); + auto headOutcome = s_s3Client->HeadObject(head); + AWS_ASSERT_SUCCESS(headOutcome); + EXPECT_EQ(static_cast(size), headOutcome.GetResult().GetContentLength()); + EXPECT_EQ("text/plain", headOutcome.GetResult().GetContentType()); + + const Aws::String destPath = LocalTempPath("big-download"); + Aws::S3::Model::GetObjectRequest getRequest; + getRequest.SetBucket(s_bucketName); + getRequest.SetKey(key); + DownloadRequest downloadRequest(getRequest, destPath); + DownloadOutcome downloadOutcome = manager.Download(downloadRequest).CompletionFuture().get(); + AWS_ASSERT_SUCCESS(downloadOutcome); + EXPECT_EQ(size, FileSize(destPath)); + EXPECT_EQ(sourceHash, FileSha256(destPath)); + + Aws::FileSystem::RemoveFileIfExists(sourcePath.c_str()); + Aws::FileSystem::RemoveFileIfExists(destPath.c_str()); +} + +} // namespace