Single File Upload/Download

cmake/sdksCommon.cmake

@@ -108,6 +108,7 @@ list(APPEND SDK_TEST_PROJECT_LIST "redshift:tests/aws-cpp-sdk-redshift-integrati
 list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3:tests/aws-cpp-sdk-s3-unit-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3-crt:tests/aws-cpp-sdk-s3-crt-integration-tests")
+list(APPEND SDK_TEST_PROJECT_LIST "s3-transfer:tests/aws-cpp-sdk-s3-transfer-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3-encryption:tests/aws-cpp-sdk-s3-encryption-tests,tests/aws-cpp-sdk-s3-encryption-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "s3control:tests/aws-cpp-sdk-s3control-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "sns:tests/aws-cpp-sdk-sns-integration-tests")

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadDataReceiver.h

@@ -0,0 +1,26 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+#pragma once
+#include <aws/s3-transfer/S3Transfer_EXPORTS.h>
+#include <aws/s3-transfer/S3DownloadBuffer.h>
+
+namespace Aws {
+namespace S3 {
+namespace Transfer {
+
+/**
+ * Callback interface for zero-copy downloads. The transfer manager delivers each part of the
+ * object to OnDataReceived as it arrives, in object order. Keep a copy of the buffer to retain
+ * the bytes past this call.
+ */
+class AWS_S3_TRANSFER_API DownloadDataReceiver {
+ public:
+  virtual ~DownloadDataReceiver() = default;
+  virtual void OnDataReceived(S3DownloadBuffer buffer) = 0;
+};
+
+}
+}
+}

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadHandle.h

@@ -1,39 +1,39 @@
 /**
-* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0.
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
 #include <future>
 #include <memory>
 #include <aws/s3-transfer/DownloadResponse.h>
+#include <aws/core/utils/memory/AWSMemory.h>
 
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
-/**
- * Returned from S3TransferManager::Download to represent a single in-flight download. The
- * handle is move-only and owns the underlying transfer state.
- */
+class DownloadHandleImpl;
+
+// Move-only handle for a single in-flight download.
 class AWS_S3_TRANSFER_API DownloadHandle final {
 public:
-  DownloadHandle();
+  explicit DownloadHandle(Aws::UniquePtr<DownloadHandleImpl> impl);
   ~DownloadHandle();
+  DownloadHandle(const DownloadHandle&) = delete;
+  DownloadHandle& operator=(const DownloadHandle&) = delete;
   DownloadHandle(DownloadHandle&&) noexcept;
   DownloadHandle& operator=(DownloadHandle&&) noexcept;
 
-  /**
-   * Returns a future that resolves once the transfer finishes, succeeds, or fails.
-   */
+  // Resolves once the transfer finishes, succeeds, or fails.
   std::future<DownloadOutcome> CompletionFuture();
 
-  /**
-   * Requests cancellation of the in-flight download. Returns immediately; the future
-   * returned by CompletionFuture will resolve with a failure once the cancel takes effect.
-   */
+  // Returns immediately; the completion future resolves with a failure once the cancel takes effect.
   void Cancel();
+
+private:
+  Aws::UniquePtr<DownloadHandleImpl> m_impl;
 };
 
 

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadRequest.h

@@ -5,10 +5,11 @@
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
 #include <aws/s3-transfer/DownloadProgressListener.h>
+#include <aws/s3-transfer/DownloadDataReceiver.h>
 #include <aws/core/utils/memory/stl/AWSString.h>
 #include <aws/core/utils/memory/stl/AWSVector.h>
-#include <aws/core/utils/stream/ResponseStream.h>
 #include <aws/s3/model/GetObjectRequest.h>
+#include <cassert>
 #include <memory>
 #include <utility>
 
@@ -17,25 +18,36 @@ namespace S3 {
 namespace Transfer {
 
 /**
- * Request type for S3TransferManager::Download. Carries the inner S3 GetObjectRequest along
- * with the local destination (file path or stream factory) and any request-level progress
- * listeners. The transfer manager parallelizes large objects via ranged GETs internally.
+ * Request type for S3TransferManager::Download. The destination is either a file path or a
+ * zero-copy DownloadDataReceiver; the two constructors make that choice exclusive.
  */
 class AWS_S3_TRANSFER_API DownloadRequest final {
  public:
+  // File download.
   explicit DownloadRequest(
     Aws::S3::Model::GetObjectRequest s3Request,
     Aws::String destinationFilePath,
-    Aws::IOStreamFactory responseStreamFactory,
     Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {})
   : m_s3Request(std::move(s3Request)),
     m_destinationFilePath(std::move(destinationFilePath)),
-    m_responseStreamFactory(std::move(responseStreamFactory)),
-    m_transferListeners(std::move(transferListeners)) {}
+    m_transferListeners(std::move(transferListeners)) {
+    assert(!m_destinationFilePath.empty() && "DownloadRequest destination file path must not be empty");
+  }
+
+  // Zero-copy stream download. Each part is delivered to the receiver in object order.
+  explicit DownloadRequest(
+    Aws::S3::Model::GetObjectRequest s3Request,
+    std::shared_ptr<DownloadDataReceiver> dataReceiver,
+    Aws::Vector<std::shared_ptr<DownloadProgressListener>> transferListeners = {})
+  : m_s3Request(std::move(s3Request)),
+    m_dataReceiver(std::move(dataReceiver)),
+    m_transferListeners(std::move(transferListeners)) {
+    assert(m_dataReceiver && "DownloadRequest data receiver must not be null");
+  }
 
   inline const Aws::S3::Model::GetObjectRequest& GetS3Request() const { return m_s3Request; }
   inline const Aws::String& GetDestinationFilePath() const { return m_destinationFilePath; }
-  inline const Aws::IOStreamFactory& GetResponseStreamFactory() const { return m_responseStreamFactory; }
+  inline const std::shared_ptr<DownloadDataReceiver>& GetDownloadDataReceiver() const { return m_dataReceiver; }
   inline const Aws::Vector<std::shared_ptr<DownloadProgressListener>>& GetTransferListeners() const {
     return m_transferListeners;
   }
@@ -44,7 +56,7 @@ class AWS_S3_TRANSFER_API DownloadRequest final {
  private:
   Aws::S3::Model::GetObjectRequest m_s3Request;
   Aws::String m_destinationFilePath;
-  Aws::IOStreamFactory m_responseStreamFactory;
+  std::shared_ptr<DownloadDataReceiver> m_dataReceiver;
   Aws::Vector<std::shared_ptr<DownloadProgressListener>> m_transferListeners;
 };
 

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/DownloadResponse.h

@@ -28,11 +28,6 @@ class AWS_S3_TRANSFER_API DownloadResponse final {
     m_s3ResultHasBeenSet = true;
     m_s3Result = std::forward<GetObjectResultT>(getS3Result);
   }
-  template <typename GetObjectResultT = Aws::S3::Model::GetObjectResult>
-  DownloadResponse& WithS3Result(GetObjectResultT&& getS3Result) {
-    SetS3Result(std::forward<GetObjectResultT>(getS3Result));
-    return *this;
-  }
 
  private:
   Aws::S3::Model::GetObjectResult m_s3Result;

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3DownloadBuffer.h

@@ -0,0 +1,46 @@
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+#pragma once
+#include <aws/s3-transfer/S3Transfer_EXPORTS.h>
+#include <aws/crt/s3/S3.h>
+#include <cstdint>
+#include <memory>
+
+namespace Aws {
+namespace S3 {
+namespace Transfer {
+
+/**
+ * One part of a zero-copy download, delivered to DownloadDataReceiver::OnDataReceived. GetData()
+ * returns a cursor into CRT-owned memory; this buffer holds an owning reference so the bytes stay
+ * valid as long as this object (or a copy of it) is alive.
+ */
+class AWS_S3_TRANSFER_API S3DownloadBuffer final {
+ public:
+  // The cursor carries the per-part body delivered by the CRT; the ticket is an owning reference
+  // that keeps the underlying CRT-owned buffer alive for as long as this object (or a copy) is alive.
+  explicit S3DownloadBuffer(std::shared_ptr<Aws::Crt::S3::S3BufferTicket> ticket,
+                            Aws::Crt::ByteCursor bytes, uint64_t rangeStart) noexcept;
+
+  S3DownloadBuffer(const S3DownloadBuffer&) = default;
+  S3DownloadBuffer& operator=(const S3DownloadBuffer&) = default;
+  S3DownloadBuffer(S3DownloadBuffer&& other) noexcept;
+  S3DownloadBuffer& operator=(S3DownloadBuffer&& other) noexcept;
+  ~S3DownloadBuffer() = default;
+
+  // Cursor into CRT-owned memory; empty if the part carried no bytes.
+  inline Aws::Crt::ByteCursor GetData() const { return m_bytes; }
+  // Byte offset within the object at which this part begins.
+  inline uint64_t GetRangeStart() const { return m_rangeStart; }
+
+ private:
+  std::shared_ptr<Aws::Crt::S3::S3BufferTicket> m_ticket;
+  Aws::Crt::ByteCursor m_bytes;
+  uint64_t m_rangeStart;
+};
+
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManager.h

@@ -1,5 +1,5 @@
 /**
-* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0.
  */
 #pragma once
@@ -11,41 +11,44 @@
 #include <aws/s3-transfer/DownloadHandle.h>
 #include <aws/core/utils/memory/AWSMemory.h>
 namespace Aws {
-  namespace S3 {
-  namespace Transfer {
-  class S3TransferManagerImpl;
+namespace S3 {
+namespace Transfer {
+
+class S3TransferManagerImpl;
+
+/**
+ * Customers construct an instance directly. The manager owns the underlying CRT client; it is
+ * neither copyable nor movable.
+ */
+class AWS_S3_TRANSFER_API S3TransferManager final {
+ public:
+  explicit S3TransferManager(const S3TransferManagerConfiguration& config);
+  ~S3TransferManager();
+
+  S3TransferManager(const S3TransferManager&) = delete;
+  S3TransferManager& operator=(const S3TransferManager&) = delete;
+  S3TransferManager(S3TransferManager&&) noexcept = delete;
+  S3TransferManager& operator=(S3TransferManager&&) noexcept = delete;
 
   /**
-   * Customers construct an instance directly. The manager owns the underlying CRT
-   * client; it is movable but not copyable.
+   * Begin uploading the object described by request. Returns immediately with a handle that can be
+   * used to wait for completion or to cancel the in-flight transfer.
    */
-  class AWS_S3_TRANSFER_API S3TransferManager final {
-  public:
-    explicit S3TransferManager(const S3TransferManagerConfiguration& config);
-    ~S3TransferManager();
-
-    S3TransferManager(const S3TransferManager&) = delete;
-    S3TransferManager& operator=(const S3TransferManager&) = delete;
-
-
-    /**
-     * Begin uploading the object described by request. Returns immediately with a handle
-     * that can be used to wait for completion or to cancel the in-flight transfer.
-     */
-    UploadHandle Upload(const UploadRequest& request);
-
-    /**
-     * Begin downloading the object described by request. Returns immediately with a handle
-     * that can be used to wait for completion or to cancel the in-flight transfer.
-     */
-    DownloadHandle Download(const DownloadRequest& request);
-
-  private:
-    Aws::UniquePtr<S3TransferManagerImpl> m_impl;
-  };
-  }
-  }
-  }
+  UploadHandle Upload(const UploadRequest& request);
+
+  /**
+   * Begin downloading the object described by request. Returns immediately with a handle that can
+   * be used to wait for completion or to cancel the in-flight transfer.
+   */
+  DownloadHandle Download(const DownloadRequest& request);
+
+ private:
+  Aws::UniquePtr<S3TransferManagerImpl> m_impl;
+};
+
+}  // namespace Transfer
+}  // namespace S3
+}  // namespace Aws
 
 
 

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3TransferManagerConfiguration.h

@@ -7,20 +7,31 @@
 
 #include <aws/core/client/GenericClientConfiguration.h>
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
+#include <aws/crt/Optional.h>
+#include <aws/crt/io/TlsOptions.h>
+#include <cstdint>
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
-struct AWS_S3_TRANSFER_API S3TransferManagerConfiguration : public Aws::Client::GenericClientConfiguration {
-  using BaseClientConfigClass = Aws::Client::GenericClientConfiguration;
-
-  S3TransferManagerConfiguration(const Aws::Client::ClientConfigurationInitValues& configuration = {});
 
-  S3TransferManagerConfiguration(const char* profileName, bool shouldDisableIMDS = false);
+constexpr uint64_t DEFAULT_PART_SIZE_BYTES = 8ULL * 1024 * 1024;
+constexpr uint64_t DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES = 16ULL * 1024 * 1024;
 
-  S3TransferManagerConfiguration(bool useSmartDefaults, const char* defaultMode = "legacy", bool shouldDisableIMDS = false);
+struct AWS_S3_TRANSFER_API S3TransferManagerConfiguration final : public Aws::Client::GenericClientConfiguration {
+  using BaseClientConfigClass = Aws::Client::GenericClientConfiguration;
 
-  S3TransferManagerConfiguration(const Aws::Client::ClientConfiguration& config);
+  explicit S3TransferManagerConfiguration(const Aws::Client::ClientConfigurationInitValues& configuration = {});
+  explicit S3TransferManagerConfiguration(const char* profileName, bool shouldDisableIMDS = false);
+  explicit S3TransferManagerConfiguration(bool useSmartDefaults, const char* defaultMode = "legacy", bool shouldDisableIMDS = false);
+  explicit S3TransferManagerConfiguration(const Aws::Client::ClientConfiguration& config);
+
+  uint64_t partSize = DEFAULT_PART_SIZE_BYTES;
+  // Object size at or above which a multipart transfer is used. Must be >= partSize.
+  uint64_t multipartUploadThreshold = DEFAULT_MULTIPART_UPLOAD_THRESHOLD_BYTES;
+  // 0 means inherit the CRT's own default.
+  double throughputTargetGbps = 0.0;
+  Aws::Crt::Optional<Aws::Crt::Io::TlsConnectionOptions> tlsConnectionOptions;
 
  private:
   void LoadS3TransferManagerSpecificConfig(const Aws::String& profileName);

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/S3Transfer_EXPORTS.h

@@ -1,5 +1,5 @@
 /**
-* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0.
  */
 #pragma once

src/aws-cpp-sdk-s3-transfer/include/aws/s3-transfer/UploadHandle.h

@@ -1,40 +1,39 @@
 /**
-* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
  * SPDX-License-Identifier: Apache-2.0.
  */
 #pragma once
 #include <aws/s3-transfer/S3Transfer_EXPORTS.h>
 #include <future>
 #include <memory>
 #include <aws/s3-transfer/UploadResponse.h>
+#include <aws/core/utils/memory/AWSMemory.h>
 
 
 namespace Aws {
 namespace S3 {
 namespace Transfer {
 
-/**
- * Returned from S3TransferManager::Upload to represent a single in-flight upload. The
- * handle is move-only and owns the underlying transfer state.
- */
+class UploadHandleImpl;
+
+// Move-only handle for a single in-flight upload.
 class AWS_S3_TRANSFER_API UploadHandle final {
 public:
-  UploadHandle();
+  explicit UploadHandle(Aws::UniquePtr<UploadHandleImpl> impl);
   ~UploadHandle();
+  UploadHandle(const UploadHandle&) = delete;
+  UploadHandle& operator=(const UploadHandle&) = delete;
   UploadHandle(UploadHandle&&) noexcept;
   UploadHandle& operator=(UploadHandle&&) noexcept;
 
-
-  /**
-   * Returns a future that resolves once the transfer finishes, succeeds, or fails.
-   */
+  // Resolves once the transfer finishes, succeeds, or fails.
   std::future<UploadOutcome> CompletionFuture();
 
-  /**
-   * Requests cancellation of the in-flight upload. Returns immediately; the future
-   * returned by CompletionFuture will resolve with a failure once the cancel takes effect.
-   */
+  // Returns immediately; the completion future resolves with a failure once the cancel takes effect.
   void Cancel();
+
+private:
+  Aws::UniquePtr<UploadHandleImpl> m_impl;
 };