diff --git a/CMakeLists.txt b/CMakeLists.txt index a907b35c2..f77ca58f9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -195,6 +195,10 @@ file(GLOB AWS_CRT_CBOR_HEADERS "include/aws/crt/cbor/*.h" ) +file(GLOB AWS_CRT_S3_HEADERS + "include/aws/crt/s3/*.h" +) + file(GLOB AWS_CRT_PUBLIC_HEADERS ${AWS_CRT_HEADERS} ${AWS_CRT_AUTH_HEADERS} @@ -206,6 +210,7 @@ file(GLOB AWS_CRT_PUBLIC_HEADERS ${AWS_CRT_HTTP_HEADERS} ${AWS_CRT_ENDPOINT_HEADERS} ${AWS_CRT_CBOR_HEADERS} + ${AWS_CRT_S3_HEADERS} ) if(BUILD_DEPS) @@ -257,6 +262,10 @@ file(GLOB AWS_CRT_CBOR_SRC "source/cbor/*.cpp" ) +file(GLOB AWS_CRT_S3_SRC + "source/s3/*.cpp" +) + file(GLOB AWS_CRT_CPP_SRC ${AWS_CRT_SRC} ${AWS_CRT_AUTH_SRC} @@ -268,6 +277,7 @@ file(GLOB AWS_CRT_CPP_SRC ${AWS_CRT_HTTP_SRC} ${AWS_CRT_ENDPOINTS_SRC} ${AWS_CRT_CBOR_SRC} + ${AWS_CRT_S3_SRC} ) if(WIN32) @@ -282,6 +292,7 @@ if(WIN32) source_group("Header Files\\aws\\crt\\http" FILES ${AWS_CRT_HTTP_HEADERS}) source_group("Header Files\\aws\\crt\\endpoints" FILES ${AWS_CRT_ENDPOINT_HEADERS}) source_group("Header Files\\aws\\crt\\cbor" FILES ${AWS_CRT_CBOR_HEADERS}) + source_group("Header Files\\aws\\crt\\s3" FILES ${AWS_CRT_S3_HEADERS}) source_group("Source Files" FILES ${AWS_CRT_SRC}) source_group("Source Files\\auth" FILES ${AWS_CRT_AUTH_SRC}) @@ -293,6 +304,7 @@ if(WIN32) source_group("Source Files\\http" FILES ${AWS_CRT_HTTP_SRC}) source_group("Source Files\\endpoints" FILES ${AWS_CRT_ENDPOINTS_SRC}) source_group("Source Files\\cbor" FILES ${AWS_CRT_CBOR_SRC}) + source_group("Source Files\\s3" FILES ${AWS_CRT_S3_SRC}) endif() endif() @@ -385,6 +397,7 @@ install(FILES ${AWS_CRT_MQTT_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/a install(FILES ${AWS_CRT_HTTP_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/aws/crt/http" COMPONENT Development) install(FILES ${AWS_CRT_ENDPOINT_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/aws/crt/endpoints" COMPONENT Development) install(FILES ${AWS_CRT_CBOR_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/aws/crt/cbor" COMPONENT Development) +install(FILES ${AWS_CRT_S3_HEADERS} DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/aws/crt/s3" COMPONENT Development) install( TARGETS ${PROJECT_NAME} diff --git a/include/aws/crt/s3/S3.h b/include/aws/crt/s3/S3.h new file mode 100644 index 000000000..e47eb4ff5 --- /dev/null +++ b/include/aws/crt/s3/S3.h @@ -0,0 +1,1235 @@ +#pragma once +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +struct aws_s3_client; +struct aws_s3_client_config; +struct aws_s3_meta_request; +struct aws_s3_meta_request_options; +struct aws_retry_strategy; + +namespace Aws +{ + namespace Crt + { + namespace Auth + { + class ICredentialsProvider; + } + + namespace S3 + { + /** + * The kind of S3 operation that a meta request will perform. + * Default is a single-request pass-through; the other values enable the + * CRT's multi-part orchestration for the named operation. + */ + // Mapped to their aws-c-s3 counterparts by a switch in S3.cpp. + enum class S3MetaRequestType + { + Default, + GetObject, + PutObject, + CopyObject, + }; + + /** + * Where a calculated request-side checksum is placed on the wire: None adds + * no payload checksum, Header puts the checksum in the request headers, and + * Trailer aws-chunked-encodes the payload and puts the checksum in the trailer. + */ + enum class S3ChecksumLocation + { + None, + Header, + Trailer, + }; + + /** + * The checksum algorithm used for request-side checksum calculation or + * response validation. A subset of the algorithms aws-c-s3 supports. + */ + enum class S3ChecksumAlgorithm + { + None, + Crc32c, + Crc32, + Sha1, + Sha256, + Crc64Nvme, + Sha512, + XXHash64, + XXHash3_64, + XXHash3_128, + }; + + /** + * Whether the client establishes connections over TLS. The + * client only consults the configured TLS connection options when + * the mode is Enabled. + */ + enum class S3TlsMode + { + Enabled, + Disabled, + }; + + /** + * How S3MetaRequestOptions::SetRecvFilepath opens the destination + * file. Mirrors aws_s3_recv_file_options. Only meaningful when a + * receive filepath is configured; the enum values are mapped to the + * C enum by a switch in S3.cpp. + */ + enum class S3RecvFileMode + { + /** Create the file, or replace it if it already exists. */ + CreateOrReplace, + /** Always create a new file; fail if the path already exists. */ + CreateNew, + /** Create a new file if it doesn't exist; otherwise append. */ + CreateOrAppend, + /** + * Write to an existing file at the position supplied via + * SetRecvFilePosition; the file must already exist. + */ + WriteToPosition, + }; + + /** + * Which network-level retry strategy the S3 client uses. Mirrors the + * flavors aws-c-io exposes. Default leaves the choice to the CRT, + * which builds its own standard strategy when none is supplied. + */ + enum class S3RetryStrategyType + { + /** Leave retry_strategy unset; the CRT builds its default strategy. */ + Default, + /** Token-bucket standard strategy (aws_retry_strategy_new_standard). */ + Standard, + /** Exponential-backoff strategy (aws_retry_strategy_new_exponential_backoff). */ + ExponentialBackoff, + /** Disable retries (aws_retry_strategy_new_no_retry). */ + NoRetry, + }; + + /** + * Tuning knobs for S3RetryStrategyType::ExponentialBackoff (the parameter + * type of CreateExponentialBackoff and the options argument to + * SetRetryStrategy). These knobs are applied only to the + * exponential-backoff strategy; they are ignored for the Standard and + * NoRetry flavors. + * + * NOTE: maxRetries is a count, not an on/off switch. A value of 0 does + * NOT disable retries -- aws-c-io treats 0 as "unset" and substitutes + * its own default (currently 5). To disable retries entirely, select + * S3RetryStrategyType::NoRetry rather than setting maxRetries to 0. + */ + struct AWS_CRT_CPP_API S3RetryStrategyExponentialBackoffOptions + { + /** + * Maximum number of retries (1..63). 0 is treated as "unset" and + * uses the aws-c-io default (currently 5), not zero retries; for + * zero retries use S3RetryStrategyType::NoRetry. + */ + size_t maxRetries = 0; + /** Scaling factor for the backoff, in milliseconds. */ + uint32_t scaleFactorMs = 500; + /** Maximum delay between retries, in seconds. */ + uint32_t maxBackoffSecs = 20; + }; + + /** + * C++ binding over a CRT aws_retry_strategy handle. Owns the handle and + * releases it on destruction; construct one with the static Create* + * factories. Hand the result to S3ClientConfig::SetRetryStrategy or + * return it from a SetRetryStrategyFactory callback - the S3 client + * acquires its own reference at construction, so the binding may be + * destroyed afterward. + * + * Move-only: the underlying handle has single-owner semantics here. + */ + class AWS_CRT_CPP_API S3RetryStrategy final + { + public: + S3RetryStrategy(const S3RetryStrategy &) = delete; + S3RetryStrategy &operator=(const S3RetryStrategy &) = delete; + S3RetryStrategy(S3RetryStrategy &&other) noexcept = default; + S3RetryStrategy &operator=(S3RetryStrategy &&other) noexcept = default; + ~S3RetryStrategy() noexcept = default; + + /** + * Build a token-bucket standard retry strategy. + * + * @return a binding owning the new strategy, or an empty binding on failure. + */ + static S3RetryStrategy CreateStandard() noexcept; + + /** + * Build an exponential-backoff retry strategy. Requires an event + * loop group to schedule the backoff timers on. + * + * @param elGroup event loop group used to schedule retries. + * @param options backoff tuning knobs. + * @return a binding owning the new strategy, or an empty binding on failure. + */ + static S3RetryStrategy CreateExponentialBackoff( + Io::EventLoopGroup &elGroup, + const S3RetryStrategyExponentialBackoffOptions &options = {}) noexcept; + + /** + * Build a strategy that disables retries. + * + * @return a binding owning the new strategy, or an empty binding on failure. + */ + static S3RetryStrategy CreateNoRetry() noexcept; + + /** + * @return true if this binding owns a valid strategy handle. + */ + explicit operator bool() const noexcept { return m_strategy != nullptr; } + + /// @private + /// Takes ownership of an already-created handle (or nullptr). + explicit S3RetryStrategy(aws_retry_strategy *strategy) noexcept; + + /// @private + aws_retry_strategy *GetUnderlyingHandle() const noexcept { return m_strategy.get(); } + + private: + ScopedResource m_strategy; + }; + + class S3Client; + class S3MetaRequest; + class S3MetaRequestOptions; + + /** + * Result delivered to FinishCallback when a meta request terminates. + * Mirrors aws_s3_meta_request_result. errorResponseHeaders is a deep copy + * you may retain freely. errorResponseBody is a borrowed view into + * CRT-owned memory, valid only for the duration of the FinishCallback + * invocation; copy out its contents if you need to retain them. + */ + struct AWS_CRT_CPP_API S3MetaRequestResult + { + /** AWS_ERROR_SUCCESS on success, a CRT error code otherwise. */ + int errorCode; + + /** HTTP status of the request (the successful response on success, or the failed request on an S3 error + * response); 0 for other error codes. */ + int responseStatus; + + /** Headers of the S3 error response. Empty if not applicable. */ + Vector errorResponseHeaders; + + /** + * Bytes of the S3 error response body, or an empty cursor if not + * applicable. Borrowed: the underlying buffer is owned by the meta + * request and freed when the FinishCallback returns, so copy the + * bytes out if you need to retain them. + */ + ByteCursor errorResponseBody; + + /** True if the server-side checksum was validated against a calculated value. */ + bool didValidateChecksum; + + /** Algorithm used for checksum validation, when didValidateChecksum is true. */ + S3ChecksumAlgorithm validationAlgorithm; + }; + + /** + * Per-meta-request checksum configuration. The same object is used to + * describe request-side checksum calculation for uploads and + * response-side checksum validation for downloads; not every field + * applies in both directions. + */ + class AWS_CRT_CPP_API S3ChecksumConfig final + { + public: + S3ChecksumConfig() noexcept; + + /** + * Set where a calculated checksum will appear on the wire. + * + * @param location the wire location for the request-side checksum. + * @return this object, to allow chaining. + */ + S3ChecksumConfig &SetLocation(S3ChecksumLocation location) noexcept + { + m_location = location; + return *this; + } + + /** + * Set the algorithm used to compute checksums for this request. + * + * @param algorithm the checksum algorithm to use. + * @return this object, to allow chaining. + */ + S3ChecksumConfig &SetChecksumAlgorithm(S3ChecksumAlgorithm algorithm) noexcept + { + m_algorithm = algorithm; + return *this; + } + + /** + * Enable or disable validation of the response-side checksum. + * Only meaningful for downloads. + * + * @param validate true to enable response checksum validation. + * @return this object, to allow chaining. + */ + S3ChecksumConfig &SetValidateResponseChecksum(bool validate) noexcept + { + m_validateResponseChecksum = validate; + return *this; + } + + /** + * @return the configured wire location for the request-side checksum. + */ + S3ChecksumLocation GetLocation() const noexcept { return m_location; } + + /** + * @return the configured checksum algorithm. + */ + S3ChecksumAlgorithm GetChecksumAlgorithm() const noexcept { return m_algorithm; } + + /** + * @return whether response-side checksum validation is enabled. + */ + bool GetValidateResponseChecksum() const noexcept { return m_validateResponseChecksum; } + + private: + S3ChecksumLocation m_location; + S3ChecksumAlgorithm m_algorithm; + bool m_validateResponseChecksum; + }; + + /** + * Configuration object used to construct an S3Client. The credentials + * provider is required at construction; every other setting is optional + * and set through the fluent Set* setters. The object owns backing + * storage for string-valued fields, the credentials provider reference, + * and the internally-built signing config, so it must outlive the + * S3Client construction call. + */ + class AWS_CRT_CPP_API S3ClientConfig final + { + public: + S3ClientConfig(const S3ClientConfig &) = delete; + S3ClientConfig(S3ClientConfig &&) = delete; + S3ClientConfig &operator=(const S3ClientConfig &) = delete; + S3ClientConfig &operator=(S3ClientConfig &&) = delete; + + /** + * Construct a config. The credentials provider is required (no + * defensible default - the SDK can't guess credentials). The + * signing config is built internally via + * S3Client::MakeDefaultSigningConfig, which wraps + * aws_s3_init_default_signing_config and additionally clears + * use_double_uri_encode and should_normalize_uri_path (both + * required for S3 request signing; without either, keys with + * reserved characters or "//" fail with SignatureDoesNotMatch). + * Region defaults to "us-east-1"; override via SetRegion. + * S3 Express is enabled by default; override via + * SetEnableS3Express. + * + * @param credentialsProvider the credentials provider used to + * source SigV4 credentials. The config retains a shared + * reference; the provider must remain valid at least + * until the S3Client is constructed (aws_s3_client_new + * deep-copies the signing config and acquires its own + * reference to the provider). + */ + explicit S3ClientConfig( + const std::shared_ptr &credentialsProvider) noexcept; + + // Defined out-of-line in S3.cpp where Impl is a complete type + // (required to destroy the ScopedResource pImpl member). + ~S3ClientConfig() noexcept; + + /** + * Override the AWS region the client signs requests for. Defaults + * to "us-east-1" if not set. + * + * @param region the region string, ex. "us-east-1". + * @return this object, to allow chaining. + */ + S3ClientConfig &SetRegion(const Crt::String ®ion) noexcept; + + /** + * Set the target aggregate throughput the client will tune toward, + * in gigabits per second. This drives the CRT's pool sizing. + * + * @param gbps target throughput in Gbps. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetThroughputTargetGbps(double gbps) noexcept; + + /** + * Set the target part size for multipart transfers (downloads and + * uploads). For uploads, the CRT may increase the effective part size + * if the target would require more than 10,000 parts (the S3 service + * limit), and will also raise it to the 5 MiB minimum upload part size. + * + * @param bytes target part size in bytes. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetPartSize(uint64_t bytes) noexcept; + + /** + * @param bytes threshold in bytes above which an upload is + * split into a multipart upload. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetMultipartUploadThreshold(uint64_t bytes) noexcept; + + /** + * Set the upper bound on memory the CRT may buffer for in-flight + * parts across all meta requests created with this client. + * + * @param bytes memory ceiling in bytes. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetMemoryLimit(uint64_t bytes) noexcept; + + /** + * Set the client bootstrap used for connection establishment. If + * not set, the process-global default bootstrap is used (see + * Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap). + * + * @param bootstrap the client bootstrap. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetClientBootstrap(Io::ClientBootstrap &bootstrap) noexcept; + + /** + * @param timeoutMs connection timeout in milliseconds. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetConnectTimeoutMs(uint32_t timeoutMs) noexcept; + + /** + * Enable read backpressure and set the initial read window, in + * bytes, for downloads. When backpressure is enabled the CRT pauses + * delivering body data once the window is exhausted until the + * application increments it. A window of 0 with backpressure + * enabled means no data is read until the window is incremented. + * + * @param enable whether to enable read backpressure. + * @param initialReadWindow initial read window in bytes. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetReadBackpressure(bool enable, uint64_t initialReadWindow) noexcept; + + /** + * Enable or disable S3 Express support. Enabled by default to + * match the AWS SDK S3 client. When enabled and no provider + * factory is set, the CRT creates a default S3 Express provider. + * + * @param enable whether to enable S3 Express support. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetEnableS3Express(bool enable) noexcept; + + /** + * Set whether the client establishes connections over TLS. The + * configured TLS connection options are only consulted when the + * mode is Enabled. + * + * @param mode the TLS mode. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetTlsMode(S3TlsMode mode) noexcept; + + /** + * Set the TLS connection options used for connections. Pair with + * SetTlsMode(S3TlsMode::Enabled); the options are ignored when TLS + * mode is Disabled. The config owns a copy of the options. + * + * @param options the TLS connection options. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetTlsConnectionOptions(const Io::TlsConnectionOptions &options) noexcept; + + /** + * Set the HTTP proxy options used for connections. The config owns + * a copy of the options. + * + * @param proxyOptions the proxy options. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetProxyOptions(const Http::HttpClientConnectionProxyOptions &proxyOptions) noexcept; + + /** + * Configure TCP keep-alive for connections. Both values are in + * seconds; a value of 0 leaves the corresponding setting at the + * operating-system default. + * + * @param keepAliveIntervalSec interval between keep-alive probes. + * @param keepAliveTimeoutSec timeout before a connection is considered dead. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetTcpKeepAlive(uint16_t keepAliveIntervalSec, uint16_t keepAliveTimeoutSec) noexcept; + + /** + * Enable connection health monitoring. A connection is shut down if + * its throughput falls below the minimum for longer than the + * allowable failure interval. + * + * @param minimumThroughputBytesPerSecond minimum acceptable throughput. + * @param allowableThroughputFailureIntervalSeconds grace interval in seconds. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetConnectionMonitoring( + uint64_t minimumThroughputBytesPerSecond, + uint32_t allowableThroughputFailureIntervalSeconds) noexcept; + + /** + * Set the network interface names the client distributes its + * connections across, allowing it to saturate multiple NICs. If + * any interface name is invalid or its link goes down, you will + * see connection failures. + * + * Experimental and only supported on Linux, macOS, and platforms + * with SO_BINDTODEVICE or IP_BOUND_IF; not supported on Windows + * (AWS_ERROR_PLATFORM_NOT_SUPPORTED is raised on unsupported + * platforms). The config owns a copy of the names. + * + * @param networkInterfaceNames the interface names, ex. "eth0". + * @return this object, to allow chaining. + */ + S3ClientConfig &SetNetworkInterfaceNames(const Vector &networkInterfaceNames) noexcept; + + /** + * Select a network-level retry strategy by flavor. This is the + * common path: the matching aws_retry_strategy is built when the + * S3Client is constructed (the point at which the event loop group + * is known). S3RetryStrategyType::Default leaves the choice to the + * CRT. Overrides any factory set via SetRetryStrategyFactory. + * + * @param type the retry strategy flavor. + * @param options backoff knobs, used only for ExponentialBackoff. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetRetryStrategy( + S3RetryStrategyType type, + const S3RetryStrategyExponentialBackoffOptions &options = {}) noexcept; + + /** + * Install a factory for fine-grained control over the retry + * strategy. Invoked once, at S3Client construction, with this + * config; it returns the S3RetryStrategy binding the client should + * use. Use this when the flavors exposed by SetRetryStrategy are + * not sufficient. Overrides any flavor set via SetRetryStrategy. + * + * @param factory callback that produces the retry strategy. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetRetryStrategyFactory( + std::function factory) noexcept; + + /** + * Install a callback invoked once the underlying CRT client has + * finished its asynchronous shutdown. aws_s3_client_release (called + * from ~S3Client) only starts teardown; the client's threads and + * connection pool may still be winding down after the destructor + * returns. This callback is the signal that teardown is complete and + * it is safe to release dependencies the client borrowed (event loop + * group, client bootstrap, credentials provider). It fires on a CRT + * thread, after the S3Client object may already be gone. + * + * @param callback the callback to invoke on shutdown completion. + * @return this object, to allow chaining. + */ + S3ClientConfig &SetClientShutdownCallback(std::function callback) noexcept; + + /// @private + /// Raw handle for the C layer (aws_s3_client_new); not part of the + /// public API. + struct aws_s3_client_config *GetUnderlyingHandle() const noexcept; + + /** + * @return the configured retry-strategy flavor (Default if unset). + */ + S3RetryStrategyType GetRetryStrategyType() const noexcept { return m_retryStrategyType; } + + /** + * @return the exponential-backoff tuning knobs; only meaningful when + * the strategy type is ExponentialBackoff. + */ + const S3RetryStrategyExponentialBackoffOptions &GetRetryStrategyOptions() const noexcept + { + return m_retryStrategyOptions; + } + + /** + * @return the retry-strategy factory, or an empty function if none + * was installed via SetRetryStrategyFactory. + */ + const std::function &GetRetryStrategyFactory() const noexcept + { + return m_retryStrategyFactory; + } + + /** + * @return the configured network interface names, or an empty vector + * if none were set. + */ + const Vector &GetNetworkInterfaceNames() const noexcept { return m_networkInterfaceNames; } + + /** + * @return the client-shutdown callback, or an empty function if none + * was installed via SetClientShutdownCallback. + */ + const std::function &GetClientShutdownCallback() const noexcept + { + return m_clientShutdownCallback; + } + + private: + // The CRT C-struct storage (aws_s3_client_config plus the proxy, + // TCP keep-alive, and connection-monitoring option backings) lives + // by value inside Impl, defined in S3.cpp. This keeps the C headers + // out of this public header and collapses what were four separate + // heap allocations into the single Impl allocation. + struct Impl; + ScopedResource m_impl; + String m_region; + Optional m_tlsConnectionOptions; + Optional m_proxyOptions; + Vector m_networkInterfaceNames; + + // Strategy is materialized at S3Client construction; the + // factory (when set) takes precedence over the enum. + S3RetryStrategyType m_retryStrategyType; + S3RetryStrategyExponentialBackoffOptions m_retryStrategyOptions; + std::function m_retryStrategyFactory; + + std::function m_clientShutdownCallback; + std::shared_ptr m_credentialsProvider; + // Held by value (no heap): single-owner, scope-bound to this + // config. Optional carries the "no signing config" state the + // ctor relies on when no credentials provider is supplied. + Optional m_signingConfig; + }; + + /** + * Abstract base for the four concrete options types. Users obtain + * a unique_ptr through the subclass's static Create factory (some + * subclasses overload Create so the body sink or source is chosen + * by argument type), then set cross-cutting fields on the base + * through the shared Set* setters before handing the object to + * S3Client::MakeMetaRequest. + * + * Mutual exclusion of body-delivery paths (callback / callback_ex + * / recv_filepath) is enforced structurally by the factory shapes. + */ + class AWS_CRT_CPP_API S3MetaRequestOptions + { + public: + /** + * Invoked once per delivered body chunk during a download. + * @param body the chunk of object bytes for this delivery. + * @param rangeStart byte offset within the object that this chunk starts at. + * @return AWS_OP_SUCCESS to continue, or an error code to abort the meta request. + */ + using BodyCallback = std::function; + + /** + * Like BodyCallback, but the chunk is delivered with a borrowed + * S3BufferTicket. The receiver may call ticket.Acquire() to extend + * the buffer's lifetime past the callback return for zero-copy + * consumption. + * @param body the chunk of object bytes for this delivery. + * @param rangeStart byte offset within the object that this chunk starts at. + * @param ticket borrowed handle to the CRT-owned buffer holding body. + * @return AWS_OP_SUCCESS to continue, or an error code to abort the meta request. + */ + using BodyCallbackEx = std::function; + + /** + * Invoked once when response headers are available. + * @param headers materialized snapshot of the response headers. + * @param responseStatus HTTP status code from the response. + * @return AWS_OP_SUCCESS to continue, or an error code to abort the meta request. + */ + using HeadersCallback = std::function &headers, int responseStatus)>; + + /** + * Invoked periodically as bytes flow. + * @param bytesTransferred number of bytes since the last invocation. + * @param contentLength total length of the transfer in bytes, if known. + */ + using ProgressCallback = std::function; + + /** + * Invoked once when the meta request terminates (success or failure). + * @param result final state of the meta request, including error + * code, HTTP status, and (on failure) the S3 error response + * headers and body. The error response body is a borrowed + * cursor into CRT-owned memory, valid only for the duration + * of this callback; copy it out if you need it. + */ + using FinishCallback = std::function; + + /** + * Invoked after the CRT has fully torn down the meta request and + * will not invoke any further callbacks. The right place to free + * any heap state that the callbacks depended on. + */ + using ShutdownCallback = std::function; + + S3MetaRequestOptions(const S3MetaRequestOptions &) = delete; + S3MetaRequestOptions(S3MetaRequestOptions &&) = delete; + S3MetaRequestOptions &operator=(const S3MetaRequestOptions &) = delete; + S3MetaRequestOptions &operator=(S3MetaRequestOptions &&) = delete; + + virtual ~S3MetaRequestOptions() noexcept; + + /** + * Override the client-level signing config for this meta request. + * + * Only the underlying handle is borrowed; no copy is taken + * (AwsSigningConfig is non-copyable). The supplied config must + * outlive the call that consumes it - S3Client::MakeMetaRequest, + * which deep-copies the signing config internally. + * + * @param config the SigV4 signing config to use for this request. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetSigningConfig(const Auth::AwsSigningConfig &config) noexcept; + + /** + * Set the checksum configuration for this meta request. + * + * @param config the checksum configuration. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetChecksumConfig(const S3ChecksumConfig &config) noexcept; + + /** + * Set a per-request endpoint override. Overrides the scheme and port + * the meta request would otherwise derive from the HTTP request. If + * the HTTP request already carries a Host header it must match this + * endpoint's authority, otherwise MakeMetaRequest fails. The options + * object owns a parsed copy of the URI for its lifetime. An invalid + * endpoint URI is recorded and causes MakeMetaRequest to fail with + * the parse error, rather than being silently ignored. + * + * @param endpoint the endpoint URI to use for this meta request. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetEndpoint(const Io::Uri &endpoint) noexcept; + + /** + * Install the response-headers callback. + * + * @param cb the callback to invoke when response headers arrive. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetHeadersCallback(HeadersCallback cb) noexcept; + + /** + * Install the progress callback. + * + * @param cb the callback to invoke as bytes flow. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetProgressCallback(ProgressCallback cb) noexcept; + + /** + * Install the finish callback. Invoked once when the meta request + * terminates. + * + * @param cb the callback to invoke on completion or failure. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetFinishCallback(FinishCallback cb) noexcept; + + /** + * Install the shutdown callback. Invoked after the CRT has fully + * torn down the meta request; the safe point to free callback + * state. + * + * @param cb the callback to invoke after final teardown. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetShutdownCallback(ShutdownCallback cb) noexcept; + + /** + * Choose how the destination file supplied to a receive-to-file + * factory is opened. Defaults to CreateOrReplace. Ignored when + * the meta request has no receive filepath (i.e. a callback sink). + * + * @param mode the file-open policy. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetRecvFileMode(S3RecvFileMode mode) noexcept; + + /** + * Set the byte offset the CRT writes at when the recv file + * mode is WriteToPosition. Ignored otherwise. + * + * @param position byte offset within the file. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetRecvFilePosition(uint64_t position) noexcept; + + /** + * Delete the receive file if the meta request fails. Off by + * default; the file is left as-is on failure. Useful when the + * caller wants the CRT to clean up its own temp file rather + * than doing the delete after the finish callback. + * + * @param deleteOnFailure whether to delete the file on failure. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetRecvFileDeleteOnFailure(bool deleteOnFailure) noexcept; + + /** + * Hint the size of the object being uploaded or downloaded. + * Used by the CRT to pick a strategy and validate part counts + * without an extra HeadObject roundtrip. Pass 0 to clear. + * + * @param bytes known object size in bytes; 0 to clear. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetObjectSizeHint(uint64_t bytes) noexcept; + + /** + * Override the client-level target part size for just this + * meta request. Same semantics as S3ClientConfig::SetPartSize. + * 0 means inherit from the client. + * + * @param bytes target part size in bytes. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetPartSize(uint64_t bytes) noexcept; + + /** + * Override the client-level multipart-upload threshold for just + * this meta request. Same semantics as + * S3ClientConfig::SetMultipartUploadThreshold. 0 means inherit + * from the client. + * + * @param bytes threshold in bytes. + * @return this object, to allow chaining. + */ + S3MetaRequestOptions &SetMultipartUploadThreshold(uint64_t bytes) noexcept; + + /// @private + /// Raw handle for the C layer (aws_s3_client_make_meta_request); not + /// part of the public API. + struct aws_s3_meta_request_options *GetUnderlyingHandle() const noexcept; + + /** @return the installed body callback, or an empty function if unset. */ + const BodyCallback &GetBodyCallback() const noexcept { return m_bodyCb; } + /** @return the installed zero-copy body callback, or an empty function if unset. */ + const BodyCallbackEx &GetBodyCallbackEx() const noexcept { return m_bodyCbEx; } + /** @return the installed response-headers callback, or an empty function if unset. */ + const HeadersCallback &GetHeadersCallback() const noexcept { return m_headersCb; } + /** @return the installed progress callback, or an empty function if unset. */ + const ProgressCallback &GetProgressCallback() const noexcept { return m_progressCb; } + /** @return the installed finish callback, or an empty function if unset. */ + const FinishCallback &GetFinishCallback() const noexcept { return m_finishCb; } + /** @return the installed shutdown callback, or an empty function if unset. */ + const ShutdownCallback &GetShutdownCallback() const noexcept { return m_shutdownCb; } + + /** + * @return a validation error recorded by a Set* setter (ex. an + * invalid endpoint URI), or AWS_ERROR_SUCCESS if none. Checked + * by MakeMetaRequest before the request is issued. + */ + int GetLastError() const noexcept { return m_lastError; } + + protected: + /** + * Protected ctor. Only invocable by subclasses. Allocates the C + * options struct, sets its type and message, and retains a + * shared reference to the HTTP request so the underlying + * aws_http_message stays alive for the meta request's lifetime. + * + * @param type the operation the CRT should orchestrate. + * @param request the prepared HTTP request. + */ + S3MetaRequestOptions( + S3MetaRequestType type, + const std::shared_ptr &request) noexcept; + + // The CRT C-struct storage (aws_s3_meta_request_options plus the + // checksum-config backing) lives by value inside Impl, defined in + // S3.cpp. Keeps the C headers out of this public header and + // collapses two heap allocations into the single Impl allocation. + // Protected so subclass ctors can reach it (through m_impl) to + // populate shape-specific fields directly rather than via a setter. + struct Impl; + ScopedResource m_impl; + std::shared_ptr m_httpRequest; + String m_operationName; + String m_sendFilepath; + String m_recvFilepath; + // Endpoint override parsed in place (no heap). aws_uri's internal + // cursors point into its own buffer, so it must never be moved + // after parsing - hence a value member parsed directly, not a + // relocatable wrapper. m_endpointInit tracks whether it holds a + // parsed URI needing cleanup; m_options->endpoint being non-null + // is the "set" signal. + aws_uri m_endpoint; + bool m_endpointInit; + // Backs the borrowed pointer the CRT holds via object_size_hint. + Optional m_objectSizeHint; + BodyCallback m_bodyCb; + BodyCallbackEx m_bodyCbEx; + HeadersCallback m_headersCb; + ProgressCallback m_progressCb; + FinishCallback m_finishCb; + ShutdownCallback m_shutdownCb; + // Sticky validation error set by a Set* setter, surfaced at + // MakeMetaRequest (mirrors the MqttClient builder's LastError()). + int m_lastError = AWS_ERROR_SUCCESS; + }; + + /** + * Options for a GetObject meta request. Every download must land + * somewhere; the three Create overloads pin the delivery path at + * construction and make it impossible to submit a GetObject that + * silently drops the body. Which sink is used is determined by + * the argument type: a BodyCallback, a BodyCallbackEx, or a + * destination file path. + */ + class AWS_CRT_CPP_API S3GetObjectMetaRequestOptions final : public S3MetaRequestOptions + { + public: + S3GetObjectMetaRequestOptions(const S3GetObjectMetaRequestOptions &) = delete; + S3GetObjectMetaRequestOptions(S3GetObjectMetaRequestOptions &&) = delete; + S3GetObjectMetaRequestOptions &operator=(const S3GetObjectMetaRequestOptions &) = delete; + S3GetObjectMetaRequestOptions &operator=(S3GetObjectMetaRequestOptions &&) = delete; + + /** + * Build options that deliver body chunks through a caller-owned + * BodyCallback. + * + * @param request the prepared HTTP request. + * @param cb the body callback to invoke for each chunk. + * @return a unique_ptr to the base type, or nullptr on failure. + */ + static ScopedResource Create( + const std::shared_ptr &request, + BodyCallback cb) noexcept; + + /** + * Build options that deliver body chunks through a caller-owned + * zero-copy BodyCallbackEx, which receives a borrowed + * S3BufferTicket for each chunk. + * + * @param request the prepared HTTP request. + * @param cb the zero-copy body callback. + * @return a unique_ptr to the base type, or nullptr on failure. + */ + static ScopedResource Create( + const std::shared_ptr &request, + BodyCallbackEx cb) noexcept; + + /** + * Build options that stream the response body directly to a + * file on disk. Neither BodyCallback nor BodyCallbackEx fires + * when this overload is used. + * + * @param request the prepared HTTP request. + * @param recvFilepath destination file path. + * @return a unique_ptr to the base type, or nullptr on failure. + */ + static ScopedResource Create( + const std::shared_ptr &request, + const Crt::String &recvFilepath) noexcept; + + /// @private Prefer the Create factories; direct construction + /// leaves the body sink unset and produces an incomplete object. + explicit S3GetObjectMetaRequestOptions(const std::shared_ptr &request) noexcept; + }; + + /** + * Options for a PutObject meta request. The body source is pinned + * at construction: either the HTTP request already carries a body + * stream (via HttpRequest::SetBody), or a source file path is + * supplied. aws-c-s3 rejects a PUT with no source. Which source + * is used is determined by whether a source file path is passed. + */ + class AWS_CRT_CPP_API S3PutObjectMetaRequestOptions final : public S3MetaRequestOptions + { + public: + S3PutObjectMetaRequestOptions(const S3PutObjectMetaRequestOptions &) = delete; + S3PutObjectMetaRequestOptions(S3PutObjectMetaRequestOptions &&) = delete; + S3PutObjectMetaRequestOptions &operator=(const S3PutObjectMetaRequestOptions &) = delete; + S3PutObjectMetaRequestOptions &operator=(S3PutObjectMetaRequestOptions &&) = delete; + + /** + * Build options for a PUT whose body is already attached to the + * HTTP request (via HttpRequest::SetBody). + * + * @param request the prepared HTTP request with body attached. + * @return a unique_ptr to the base type, or nullptr on failure. + */ + static ScopedResource Create( + const std::shared_ptr &request) noexcept; + + /** + * Build options for a PUT whose body is read from a file on disk. + * Do NOT also attach a body to the HTTP request - aws-c-s3 + * rejects requests with more than one body source. + * + * @param request the prepared HTTP request (no body attached). + * @param sendFilepath source file path. + * @return a unique_ptr to the base type, or nullptr on failure. + */ + static ScopedResource Create( + const std::shared_ptr &request, + const Crt::String &sendFilepath) noexcept; + + /** + * Build options for a PUT whose body is supplied incrementally via + * S3MetaRequest::Write. Do NOT attach a body to the HTTP request or + * supply a send filepath - aws-c-s3 rejects more than one body + * source. The object is uploaded as multipart and the content length + * need not be known up front. + * + * @param request the prepared HTTP request (no body attached). + * @return a unique_ptr to the base type, or nullptr on failure. + */ + static ScopedResource CreateWithAsyncWrites( + const std::shared_ptr &request) noexcept; + + /// @private Prefer the Create factories; direct construction + /// leaves the send filepath unset. + explicit S3PutObjectMetaRequestOptions(const std::shared_ptr &request) noexcept; + }; + + /** + * Options for a CopyObject meta request. The source is identified + * by the x-amz-copy-source header on the HTTP request; there is no + * caller-side body source or sink. + */ + class AWS_CRT_CPP_API S3CopyObjectMetaRequestOptions final : public S3MetaRequestOptions + { + public: + S3CopyObjectMetaRequestOptions(const S3CopyObjectMetaRequestOptions &) = delete; + S3CopyObjectMetaRequestOptions(S3CopyObjectMetaRequestOptions &&) = delete; + S3CopyObjectMetaRequestOptions &operator=(const S3CopyObjectMetaRequestOptions &) = delete; + S3CopyObjectMetaRequestOptions &operator=(S3CopyObjectMetaRequestOptions &&) = delete; + + /** + * Build options for a CopyObject meta request. + * + * @param request the prepared HTTP request. Must carry the + * x-amz-copy-source header identifying the source object. + * @return a unique_ptr to the base type, or nullptr on failure. + */ + static ScopedResource Create( + const std::shared_ptr &request) noexcept; + + /// @private Prefer the Create factory. + explicit S3CopyObjectMetaRequestOptions(const std::shared_ptr &request) noexcept; + }; + + /** + * Options for a S3MetaRequestType::Default meta request. Use this + * for any S3 operation that is not GetObject, PutObject, or + * CopyObject (ex. CreateBucket, HeadObject, ListObjectsV2). The + * operation name is required and must be the canonical S3 API + * operation name; mis-naming can produce incorrect behavior or + * leak sensitive data on error paths (aws-c-s3 uses the name to + * drive operation-specific response handling). + */ + class AWS_CRT_CPP_API S3DefaultObjectMetaRequestOptions final : public S3MetaRequestOptions + { + public: + S3DefaultObjectMetaRequestOptions(const S3DefaultObjectMetaRequestOptions &) = delete; + S3DefaultObjectMetaRequestOptions(S3DefaultObjectMetaRequestOptions &&) = delete; + S3DefaultObjectMetaRequestOptions &operator=(const S3DefaultObjectMetaRequestOptions &) = delete; + S3DefaultObjectMetaRequestOptions &operator=(S3DefaultObjectMetaRequestOptions &&) = delete; + + /** + * Build options for a S3MetaRequestType::Default meta request. + * + * @param request the prepared HTTP request. + * @param operationName the S3 API operation name + * (ex. "CreateBucket", "HeadObject", "ListObjectsV2"). + * @return a unique_ptr to the base type, or nullptr on failure. + */ + static ScopedResource Create( + const std::shared_ptr &request, + const Crt::String &operationName) noexcept; + + /// @private Prefer the Create factory. + S3DefaultObjectMetaRequestOptions( + const std::shared_ptr &request, + const Crt::String &operationName) noexcept; + }; + + /** + * High-throughput client for issuing S3 meta requests. Holds the + * underlying aws_s3_client, including its thread pool, connection + * pool, and signing config; the C handle is released when the wrapper + * is destroyed. A single client may serve many concurrent meta + * requests; create one per region per process unless you have a + * specific reason to do otherwise. + */ + class AWS_CRT_CPP_API S3Client final + { + public: + S3Client(const S3Client &) = delete; + S3Client(S3Client &&) = delete; + S3Client &operator=(const S3Client &) = delete; + S3Client &operator=(S3Client &&) = delete; + + explicit S3Client(const S3ClientConfig &config) noexcept; + ~S3Client() noexcept = default; + + /** + * Submit a meta request. The callbacks installed on the options + * object are copied into a CRT-owned callback bundle that backs + * the running request, so the caller may discard the options + * object after this call returns. + * + * @param options the configured options for this meta request, + * produced by one of the subclass factories. Borrowed for + * the duration of the call only; the caller retains + * ownership. + * @return a handle to the running meta request, or nullptr on + * failure. On failure, LastError() returns the CRT error + * code. + */ + std::shared_ptr MakeMetaRequest(S3MetaRequestOptions &options) noexcept; + + /** + * Populate an existing signing config with defaults appropriate + * for S3, sourced from the given region and credentials provider. + * Fills the caller's config in place rather than returning one - + * AwsSigningConfig (via ISigningConfig) deletes its copy and move + * constructors, so it cannot be returned by value, and this avoids + * a heap allocation for the common case where the caller already + * owns the config (ex. as a member or on the stack). + * + * @param config the signing config to populate. + * @param region the AWS region to sign requests for. + * @param provider the credentials provider used during signing. + * @return true on success, or false if provider is null. + */ + static bool MakeDefaultSigningConfig( + Auth::AwsSigningConfig &config, + const String ®ion, + const std::shared_ptr &provider) noexcept; + + /** + * @return true if the underlying client was constructed successfully. + */ + explicit operator bool() const noexcept { return m_client != nullptr; } + + /** + * @return the CRT error code from the most recent failed operation + * on this client, or AWS_ERROR_UNKNOWN if none has been + * recorded. + */ + int LastError() const noexcept; + + private: + ScopedResource m_client; + int m_lastError; + }; + + /** + * Handle to a single in-flight or recently-completed meta request, + * obtained from S3Client::MakeMetaRequest. Callbacks continue to + * fire correctly even if the caller drops their shared_ptr: the + * CRT's callback bundle keeps its own reference until shutdown. + */ + class AWS_CRT_CPP_API S3MetaRequest final + { + public: + S3MetaRequest(const S3MetaRequest &) = delete; + S3MetaRequest(S3MetaRequest &&) = delete; + S3MetaRequest &operator=(const S3MetaRequest &) = delete; + S3MetaRequest &operator=(S3MetaRequest &&) = delete; + + ~S3MetaRequest() noexcept = default; + + /** + * Request cancellation of the in-flight meta request. The CRT + * cancels asynchronously; the finish and shutdown callbacks will + * still fire to signal final teardown. + */ + void Cancel() noexcept; + + /** + * Move the flow-control read window forward by the given number of + * bytes. Only meaningful when the client was configured with read + * backpressure (S3ClientConfig::SetReadBackpressure): the CRT pauses + * delivering body data once the window is exhausted, and the + * application must call this to let more data flow. A typical pattern + * is to increment by the size of each chunk once it has been consumed. + * + * @param bytes number of bytes to add to the read window. + */ + void IncrementReadWindow(uint64_t bytes) noexcept; + + /** + * Write the next chunk of body data for an async-writes PUT (see + * S3PutObjectMetaRequestOptions::CreateWithAsyncWrites). The returned + * future completes with the CRT error code (0 on success) once the CRT + * is ready to accept more data. You MUST NOT call Write again until the + * previous call's future has completed. Pass eof=true for the final + * chunk and do not call Write again afterward. + * + * @param data the chunk of body bytes to send; may be any size. + * @param eof true if this is the final chunk. + * @return a future resolving to the CRT error code (0 on success). + */ + std::future Write(ByteCursor data, bool eof) noexcept; + + /** + * @return the CRT error code from the most recent failed operation + * on this meta request, or AWS_ERROR_UNKNOWN if none has + * been recorded. + */ + int LastError() const noexcept; + + /// @private + /// Callers must call SetUnderlyingHandle before publishing the + /// wrapper. + S3MetaRequest() noexcept; + + /// @private + /// Takes ownership of the C handle returned by + /// aws_s3_client_make_meta_request (or nullptr on failure). + void SetUnderlyingHandle(struct aws_s3_meta_request *handle) noexcept; + + /// @private + void SetLastError(int errorCode) noexcept { m_lastError = errorCode; } + + private: + ScopedResource m_metaRequest; + int m_lastError; + }; + + } // namespace S3 + } // namespace Crt +} // namespace Aws diff --git a/include/aws/crt/s3/S3BufferTicket.h b/include/aws/crt/s3/S3BufferTicket.h new file mode 100644 index 000000000..75c036ddf --- /dev/null +++ b/include/aws/crt/s3/S3BufferTicket.h @@ -0,0 +1,76 @@ +#pragma once +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include + +#include + +struct aws_s3_buffer_ticket; + +namespace Aws +{ + namespace Crt + { + namespace S3 + { + /** + * Owning handle to a buffer that the CRT has loaned out as part of a + * stream download. The CRT delivers each received part by invoking the + * BodyCallbackEx with a borrowed ticket pointing at memory in the CRT's + * pool. The ticket is alive only for the duration of that callback; call + * Acquire() inside the callback to obtain a shared_ptr that keeps the + * buffer valid until the last copy of it is destroyed. + */ + class AWS_CRT_CPP_API S3BufferTicket final + { + public: + S3BufferTicket(const S3BufferTicket &) = delete; + S3BufferTicket(S3BufferTicket &&) = delete; + S3BufferTicket &operator=(const S3BufferTicket &) = delete; + S3BufferTicket &operator=(S3BufferTicket &&) = delete; + + ~S3BufferTicket() noexcept = default; + + /** + * Take a new reference to the underlying buffer and hand it back as + * a shared_ptr. Copy the returned shared_ptr freely to share the + * buffer across owners; the CRT reference is released once when the + * last copy is destroyed, so acquire/release stays hidden behind the + * shared_ptr interface. + * + * @return a shared handle keeping the buffer valid until the last + * copy is destroyed. + */ + std::shared_ptr Acquire() noexcept; + + /** + * Return a cursor over the object bytes this ticket references. The + * cursor points directly into the CRT-owned buffer - reading from it + * copies nothing. It stays valid as long as this ticket (or a + * shared_ptr obtained from Acquire()) is alive. Returns an empty + * cursor if the ticket holds no buffer. + * + * @return a cursor into the buffer; empty if there is none. + */ + ByteCursor Claim() noexcept; + + /// @private + /// Wraps a borrowed C ticket handle (or nullptr) without taking a + /// reference; the wrapper never releases it. Acquire() is the only + /// path that takes and releases a reference. + explicit S3BufferTicket(struct aws_s3_buffer_ticket *ticket) noexcept; + + private: + // Borrowed, non-owning: the reference is owned by the CRT (for the + // stack ticket in the callback) or by an Acquire() shared_ptr's + // deleter - never released by this raw handle directly. + struct aws_s3_buffer_ticket *m_ticket; + }; + + } // namespace S3 + } // namespace Crt +} // namespace Aws diff --git a/source/s3/S3.cpp b/source/s3/S3.cpp new file mode 100644 index 000000000..2baa425de --- /dev/null +++ b/source/s3/S3.cpp @@ -0,0 +1,1078 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace Aws +{ + namespace Crt + { + namespace S3 + { + S3ChecksumConfig::S3ChecksumConfig() noexcept + // Trailer + CRC64NVME matches the AWS SDK default. A location + // of None disables the algorithm entirely. + : m_location(S3ChecksumLocation::Trailer), m_algorithm(S3ChecksumAlgorithm::Crc64Nvme), + m_validateResponseChecksum(false) + { + } + + S3RetryStrategy::S3RetryStrategy(aws_retry_strategy *strategy) noexcept + : m_strategy(strategy, aws_retry_strategy_release) + { + } + + S3RetryStrategy S3RetryStrategy::CreateStandard() noexcept + { + struct aws_standard_retry_options options; + AWS_ZERO_STRUCT(options); + return S3RetryStrategy(aws_retry_strategy_new_standard(ApiAllocator(), &options)); + } + + S3RetryStrategy S3RetryStrategy::CreateExponentialBackoff( + Io::EventLoopGroup &elGroup, + const S3RetryStrategyExponentialBackoffOptions &options) noexcept + { + struct aws_exponential_backoff_retry_options backoffOptions; + AWS_ZERO_STRUCT(backoffOptions); + backoffOptions.el_group = elGroup.GetUnderlyingHandle(); + // maxRetries == 0 is passed through as-is; aws-c-io reads 0 as + // "unset" and substitutes its default (5). Callers wanting zero + // retries should use S3RetryStrategyType::NoRetry instead. + backoffOptions.max_retries = options.maxRetries; + backoffOptions.backoff_scale_factor_ms = options.scaleFactorMs; + backoffOptions.max_backoff_secs = options.maxBackoffSecs; + backoffOptions.jitter_mode = AWS_EXPONENTIAL_BACKOFF_JITTER_FULL; + return S3RetryStrategy(aws_retry_strategy_new_exponential_backoff(ApiAllocator(), &backoffOptions)); + } + + S3RetryStrategy S3RetryStrategy::CreateNoRetry() noexcept + { + struct aws_no_retry_options options; + AWS_ZERO_STRUCT(options); + return S3RetryStrategy(aws_retry_strategy_new_no_retry(ApiAllocator(), &options)); + } + + // Holds all CRT C-struct storage for S3ClientConfig by value, so the + // whole set is one heap allocation (the Impl) instead of one per + // struct. Value-initialized ({}) so every C struct starts zeroed, + // matching the previous aws_mem_calloc behavior. + struct S3ClientConfig::Impl + { + aws_s3_client_config config = {}; + aws_http_proxy_options proxyOptions = {}; + aws_s3_tcp_keep_alive_options tcpKeepAlive = {}; + aws_http_connection_monitoring_options monitoring = {}; + }; + + S3ClientConfig::S3ClientConfig( + const std::shared_ptr &credentialsProvider) noexcept + : m_impl(New(ApiAllocator()), [](Impl *p) { Delete(p, ApiAllocator()); }), m_region("us-east-1"), + m_retryStrategyType(S3RetryStrategyType::Default), m_credentialsProvider(credentialsProvider) + { + m_impl->config.region = ByteCursorFromString(m_region); + + // Build the signing config in place (no heap). Left empty when + // no credentials provider is supplied; a null signing_config + // makes aws_s3_client_new fail with AWS_ERROR_INVALID_ARGUMENT, + // surfaced via LastError(). + if (credentialsProvider) + { + m_signingConfig.emplace(ApiAllocator()); + if (S3Client::MakeDefaultSigningConfig(*m_signingConfig, m_region, credentialsProvider)) + { + m_impl->config.signing_config = m_signingConfig->GetUnderlyingHandle(); + } + else + { + m_signingConfig.reset(); + } + } + + // S3 Express on by default to match the AWS SDK S3 client. + m_impl->config.enable_s3express = true; + } + + struct aws_s3_client_config *S3ClientConfig::GetUnderlyingHandle() const noexcept + { + return &m_impl->config; + } + + // Out-of-line so the ScopedResource is destroyed where Impl is + // a complete type. + S3ClientConfig::~S3ClientConfig() noexcept = default; + + S3ClientConfig &S3ClientConfig::SetRegion(const Crt::String ®ion) noexcept + { + m_region = region; + m_impl->config.region = ByteCursorFromString(m_region); + // Keep the signing config's region in sync with the client's. + if (m_signingConfig) + { + m_signingConfig->SetRegion(m_region); + } + return *this; + } + + S3ClientConfig &S3ClientConfig::SetThroughputTargetGbps(double gbps) noexcept + { + m_impl->config.throughput_target_gbps = gbps; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetPartSize(uint64_t bytes) noexcept + { + m_impl->config.part_size = bytes; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetMultipartUploadThreshold(uint64_t bytes) noexcept + { + m_impl->config.multipart_upload_threshold = bytes; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetMemoryLimit(uint64_t bytes) noexcept + { + m_impl->config.memory_limit_in_bytes = bytes; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetClientBootstrap(Io::ClientBootstrap &bootstrap) noexcept + { + m_impl->config.client_bootstrap = bootstrap.GetUnderlyingHandle(); + return *this; + } + + S3ClientConfig &S3ClientConfig::SetConnectTimeoutMs(uint32_t timeoutMs) noexcept + { + m_impl->config.connect_timeout_ms = timeoutMs; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetReadBackpressure(bool enable, uint64_t initialReadWindow) noexcept + { + m_impl->config.enable_read_backpressure = enable; + m_impl->config.initial_read_window = static_cast(initialReadWindow); + return *this; + } + + S3ClientConfig &S3ClientConfig::SetEnableS3Express(bool enable) noexcept + { + m_impl->config.enable_s3express = enable; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetTlsMode(S3TlsMode mode) noexcept + { + switch (mode) + { + case S3TlsMode::Enabled: + m_impl->config.tls_mode = AWS_MR_TLS_ENABLED; + break; + case S3TlsMode::Disabled: + m_impl->config.tls_mode = AWS_MR_TLS_DISABLED; + break; + } + return *this; + } + + S3ClientConfig &S3ClientConfig::SetTlsConnectionOptions(const Io::TlsConnectionOptions &options) noexcept + { + m_tlsConnectionOptions = options; + m_impl->config.tls_connection_options = m_tlsConnectionOptions->GetUnderlyingHandle(); + return *this; + } + + S3ClientConfig &S3ClientConfig::SetProxyOptions( + const Http::HttpClientConnectionProxyOptions &proxyOptions) noexcept + { + m_proxyOptions = proxyOptions; + m_proxyOptions->InitializeRawProxyOptions(m_impl->proxyOptions); + m_impl->config.proxy_options = &m_impl->proxyOptions; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetTcpKeepAlive( + uint16_t keepAliveIntervalSec, + uint16_t keepAliveTimeoutSec) noexcept + { + m_impl->tcpKeepAlive.keep_alive_interval_sec = keepAliveIntervalSec; + m_impl->tcpKeepAlive.keep_alive_timeout_sec = keepAliveTimeoutSec; + m_impl->config.tcp_keep_alive_options = &m_impl->tcpKeepAlive; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetConnectionMonitoring( + uint64_t minimumThroughputBytesPerSecond, + uint32_t allowableThroughputFailureIntervalSeconds) noexcept + { + m_impl->monitoring.minimum_throughput_bytes_per_second = minimumThroughputBytesPerSecond; + m_impl->monitoring.allowable_throughput_failure_interval_seconds = + allowableThroughputFailureIntervalSeconds; + m_impl->config.monitoring_options = &m_impl->monitoring; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetNetworkInterfaceNames( + const Vector &networkInterfaceNames) noexcept + { + // Cursor array is built as a local at S3Client construction + // (deep-copied there); only the backing strings live here. + m_networkInterfaceNames = networkInterfaceNames; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetRetryStrategy( + S3RetryStrategyType type, + const S3RetryStrategyExponentialBackoffOptions &options) noexcept + { + // Flavor and factory paths are mutually exclusive; the + // strategy is materialized at S3Client construction. + m_retryStrategyType = type; + m_retryStrategyOptions = options; + m_retryStrategyFactory = nullptr; + return *this; + } + + S3ClientConfig &S3ClientConfig::SetRetryStrategyFactory( + std::function factory) noexcept + { + m_retryStrategyFactory = std::move(factory); + return *this; + } + + S3ClientConfig &S3ClientConfig::SetClientShutdownCallback(std::function callback) noexcept + { + m_clientShutdownCallback = std::move(callback); + return *this; + } + + // Holds the CRT C-struct storage for S3MetaRequestOptions by value, so + // the options struct and its checksum-config backing are one heap + // allocation (the Impl) instead of two. Value-initialized ({}) so both + // start zeroed, matching the previous aws_mem_calloc behavior. + struct S3MetaRequestOptions::Impl + { + aws_s3_meta_request_options options = {}; + aws_s3_checksum_config checksum = {}; + }; + + S3MetaRequestOptions::S3MetaRequestOptions( + S3MetaRequestType type, + const std::shared_ptr &request) noexcept + : m_impl(New(ApiAllocator()), [](Impl *p) { Delete(p, ApiAllocator()); }), m_httpRequest(request), + m_endpointInit(false) + { + AWS_ZERO_STRUCT(m_endpoint); + switch (type) + { + case S3MetaRequestType::Default: + m_impl->options.type = AWS_S3_META_REQUEST_TYPE_DEFAULT; + break; + case S3MetaRequestType::GetObject: + m_impl->options.type = AWS_S3_META_REQUEST_TYPE_GET_OBJECT; + break; + case S3MetaRequestType::PutObject: + m_impl->options.type = AWS_S3_META_REQUEST_TYPE_PUT_OBJECT; + break; + case S3MetaRequestType::CopyObject: + m_impl->options.type = AWS_S3_META_REQUEST_TYPE_COPY_OBJECT; + break; + } + m_impl->options.message = request ? request->GetUnderlyingMessage() : nullptr; + } + + struct aws_s3_meta_request_options *S3MetaRequestOptions::GetUnderlyingHandle() const noexcept + { + return &m_impl->options; + } + + S3MetaRequestOptions::~S3MetaRequestOptions() noexcept + { + if (m_endpointInit) + { + aws_uri_clean_up(&m_endpoint); + } + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetSigningConfig(const Auth::AwsSigningConfig &config) noexcept + { + m_impl->options.signing_config = config.GetUnderlyingHandle(); + return *this; + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetChecksumConfig(const S3ChecksumConfig &config) noexcept + { + switch (config.GetLocation()) + { + case S3ChecksumLocation::None: + m_impl->checksum.location = AWS_SCL_NONE; + break; + case S3ChecksumLocation::Header: + m_impl->checksum.location = AWS_SCL_HEADER; + break; + case S3ChecksumLocation::Trailer: + m_impl->checksum.location = AWS_SCL_TRAILER; + break; + } + switch (config.GetChecksumAlgorithm()) + { + case S3ChecksumAlgorithm::None: + m_impl->checksum.checksum_algorithm = AWS_SCA_NONE; + break; + case S3ChecksumAlgorithm::Crc32c: + m_impl->checksum.checksum_algorithm = AWS_SCA_CRC32C; + break; + case S3ChecksumAlgorithm::Crc32: + m_impl->checksum.checksum_algorithm = AWS_SCA_CRC32; + break; + case S3ChecksumAlgorithm::Sha1: + m_impl->checksum.checksum_algorithm = AWS_SCA_SHA1; + break; + case S3ChecksumAlgorithm::Sha256: + m_impl->checksum.checksum_algorithm = AWS_SCA_SHA256; + break; + case S3ChecksumAlgorithm::Crc64Nvme: + m_impl->checksum.checksum_algorithm = AWS_SCA_CRC64NVME; + break; + case S3ChecksumAlgorithm::Sha512: + m_impl->checksum.checksum_algorithm = AWS_SCA_SHA512; + break; + case S3ChecksumAlgorithm::XXHash64: + m_impl->checksum.checksum_algorithm = AWS_SCA_XXHASH64; + break; + case S3ChecksumAlgorithm::XXHash3_64: + m_impl->checksum.checksum_algorithm = AWS_SCA_XXHASH3_64; + break; + case S3ChecksumAlgorithm::XXHash3_128: + m_impl->checksum.checksum_algorithm = AWS_SCA_XXHASH3_128; + break; + } + m_impl->checksum.validate_response_checksum = config.GetValidateResponseChecksum(); + m_impl->options.checksum_config = &m_impl->checksum; + return *this; + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetEndpoint(const Io::Uri &endpoint) noexcept + { + // Clear any previously-parsed endpoint before replacing it. + if (m_endpointInit) + { + aws_uri_clean_up(&m_endpoint); + AWS_ZERO_STRUCT(m_endpoint); + m_endpointInit = false; + } + m_impl->options.endpoint = nullptr; + + // A malformed endpoint is a hard error, never silently dropped. + // The failure is sticky and surfaced by MakeMetaRequest, since + // this fluent setter has no error return (mirrors the MqttClient + // builder's LastError() pattern). + if (!endpoint) + { + m_lastError = endpoint.LastError(); + return *this; + } + + // Parse into the value member in place: aws_uri's cursors point + // into its own buffer, so it must not be moved after parsing. The + // CRT reads host/scheme/port synchronously during MakeMetaRequest, + // so this only needs to outlive that call. + ByteCursor fullUri = endpoint.GetFullUri(); + if (aws_uri_init_parse(&m_endpoint, ApiAllocator(), &fullUri) != AWS_OP_SUCCESS) + { + m_lastError = aws_last_error(); + AWS_ZERO_STRUCT(m_endpoint); + return *this; + } + m_endpointInit = true; + m_impl->options.endpoint = &m_endpoint; + return *this; + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetHeadersCallback(HeadersCallback cb) noexcept + { + m_headersCb = std::move(cb); + return *this; + } + S3MetaRequestOptions &S3MetaRequestOptions::SetProgressCallback(ProgressCallback cb) noexcept + { + m_progressCb = std::move(cb); + return *this; + } + S3MetaRequestOptions &S3MetaRequestOptions::SetFinishCallback(FinishCallback cb) noexcept + { + m_finishCb = std::move(cb); + return *this; + } + S3MetaRequestOptions &S3MetaRequestOptions::SetShutdownCallback(ShutdownCallback cb) noexcept + { + m_shutdownCb = std::move(cb); + return *this; + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetRecvFileMode(S3RecvFileMode mode) noexcept + { + switch (mode) + { + case S3RecvFileMode::CreateOrReplace: + m_impl->options.recv_file_option = AWS_S3_RECV_FILE_CREATE_OR_REPLACE; + break; + case S3RecvFileMode::CreateNew: + m_impl->options.recv_file_option = AWS_S3_RECV_FILE_CREATE_NEW; + break; + case S3RecvFileMode::CreateOrAppend: + m_impl->options.recv_file_option = AWS_S3_RECV_FILE_CREATE_OR_APPEND; + break; + case S3RecvFileMode::WriteToPosition: + m_impl->options.recv_file_option = AWS_S3_RECV_FILE_WRITE_TO_POSITION; + break; + } + return *this; + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetRecvFilePosition(uint64_t position) noexcept + { + m_impl->options.recv_file_position = position; + return *this; + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetRecvFileDeleteOnFailure(bool deleteOnFailure) noexcept + { + m_impl->options.recv_file_delete_on_failure = deleteOnFailure; + return *this; + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetObjectSizeHint(uint64_t bytes) noexcept + { + // CRT borrows const uint64_t*, so the value must live here; + // bytes == 0 clears the hint and releases the borrowed pointer. + if (bytes == 0) + { + m_objectSizeHint.reset(); + m_impl->options.object_size_hint = nullptr; + } + else + { + m_objectSizeHint = bytes; + m_impl->options.object_size_hint = &m_objectSizeHint.value(); + } + return *this; + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetPartSize(uint64_t bytes) noexcept + { + m_impl->options.part_size = bytes; + return *this; + } + + S3MetaRequestOptions &S3MetaRequestOptions::SetMultipartUploadThreshold(uint64_t bytes) noexcept + { + m_impl->options.multipart_upload_threshold = bytes; + return *this; + } + + /***************************************************** + * + * S3GetObjectMetaRequestOptions + * + *****************************************************/ + // Allocate the subclass through the CRT allocator and wrap it in a + // unique_ptr whose deleter frees through that same allocator. The + // ScopedResource upcasts from the subclass pointer while + // retaining the correct destructor via its deleter. + template + static ScopedResource s_makeOptions(Args &&...args) noexcept + { + Allocator *allocator = ApiAllocator(); + return ScopedResource( + New(allocator, std::forward(args)...), + [allocator](S3MetaRequestOptions *p) + { + if (p != nullptr) + { + Aws::Crt::Delete(p, allocator); + } + }); + } + + // Build a SubclassT options object from request, then run configure() + // on the concrete subclass pointer to set its shape-specific fields. + // Folds the make + null-check + downcast that every configuring Create + // factory would otherwise repeat. + template + static ScopedResource s_makeConfiguredOptions( + const std::shared_ptr &request, + ConfigureFn &&configure) noexcept + { + auto opts = s_makeOptions(request); + if (opts) + { + configure(static_cast(opts.get())); + } + return opts; + } + + S3GetObjectMetaRequestOptions::S3GetObjectMetaRequestOptions( + const std::shared_ptr &request) noexcept + : S3MetaRequestOptions(S3MetaRequestType::GetObject, request) + { + } + + ScopedResource S3GetObjectMetaRequestOptions::Create( + const std::shared_ptr &request, + BodyCallback cb) noexcept + { + return s_makeConfiguredOptions( + request, [&](S3GetObjectMetaRequestOptions *self) { self->m_bodyCb = std::move(cb); }); + } + + ScopedResource S3GetObjectMetaRequestOptions::Create( + const std::shared_ptr &request, + BodyCallbackEx cb) noexcept + { + return s_makeConfiguredOptions( + request, [&](S3GetObjectMetaRequestOptions *self) { self->m_bodyCbEx = std::move(cb); }); + } + + ScopedResource S3GetObjectMetaRequestOptions::Create( + const std::shared_ptr &request, + const Crt::String &recvFilepath) noexcept + { + return s_makeConfiguredOptions( + request, + [&](S3GetObjectMetaRequestOptions *self) + { + self->m_recvFilepath = recvFilepath; + self->m_impl->options.recv_filepath = ByteCursorFromString(self->m_recvFilepath); + }); + } + + /***************************************************** + * + * S3PutObjectMetaRequestOptions + * + *****************************************************/ + S3PutObjectMetaRequestOptions::S3PutObjectMetaRequestOptions( + const std::shared_ptr &request) noexcept + : S3MetaRequestOptions(S3MetaRequestType::PutObject, request) + { + } + + ScopedResource S3PutObjectMetaRequestOptions::Create( + const std::shared_ptr &request) noexcept + { + return s_makeOptions(request); + } + + ScopedResource S3PutObjectMetaRequestOptions::Create( + const std::shared_ptr &request, + const Crt::String &sendFilepath) noexcept + { + return s_makeConfiguredOptions( + request, + [&](S3PutObjectMetaRequestOptions *self) + { + self->m_sendFilepath = sendFilepath; + self->m_impl->options.send_filepath = ByteCursorFromString(self->m_sendFilepath); + }); + } + + ScopedResource S3PutObjectMetaRequestOptions::CreateWithAsyncWrites( + const std::shared_ptr &request) noexcept + { + return s_makeConfiguredOptions( + request, + [](S3PutObjectMetaRequestOptions *self) { self->m_impl->options.send_using_async_writes = true; }); + } + + /***************************************************** + * + * S3CopyObjectMetaRequestOptions + * + *****************************************************/ + S3CopyObjectMetaRequestOptions::S3CopyObjectMetaRequestOptions( + const std::shared_ptr &request) noexcept + : S3MetaRequestOptions(S3MetaRequestType::CopyObject, request) + { + } + + ScopedResource S3CopyObjectMetaRequestOptions::Create( + const std::shared_ptr &request) noexcept + { + return s_makeOptions(request); + } + + /***************************************************** + * + * S3DefaultObjectMetaRequestOptions + * + *****************************************************/ + S3DefaultObjectMetaRequestOptions::S3DefaultObjectMetaRequestOptions( + const std::shared_ptr &request, + const Crt::String &operationName) noexcept + : S3MetaRequestOptions(S3MetaRequestType::Default, request) + { + m_operationName = operationName; + m_impl->options.operation_name = ByteCursorFromString(m_operationName); + } + + ScopedResource S3DefaultObjectMetaRequestOptions::Create( + const std::shared_ptr &request, + const Crt::String &operationName) noexcept + { + return s_makeOptions(request, operationName); + } + + struct S3MetaRequestCallbackData + { + std::shared_ptr wrapper; + S3MetaRequestOptions::BodyCallback bodyCb; + S3MetaRequestOptions::BodyCallbackEx bodyCbEx; + S3MetaRequestOptions::HeadersCallback headersCb; + S3MetaRequestOptions::ProgressCallback progressCb; + S3MetaRequestOptions::FinishCallback finishCb; + S3MetaRequestOptions::ShutdownCallback shutdownCb; + }; + + static Vector s_materializeHeaders(const struct aws_http_headers *headers) noexcept + { + Vector out; + if (headers == nullptr) + { + return out; + } + const size_t count = aws_http_headers_count(headers); + out.reserve(count); + for (size_t i = 0; i < count; ++i) + { + Http::HttpHeader header; + if (aws_http_headers_get_index(headers, i, &header) == AWS_OP_SUCCESS) + { + out.emplace_back(header); + } + } + return out; + } + + static int s_onHeaders( + struct aws_s3_meta_request * /*meta*/, + const struct aws_http_headers *headers, + int responseStatus, + void *user_data) + { + auto *data = static_cast(user_data); + if (!data->headersCb) + { + return AWS_OP_SUCCESS; + } + return data->headersCb(s_materializeHeaders(headers), responseStatus); + } + + static int s_onBody( + struct aws_s3_meta_request * /*meta*/, + const ByteCursor *body, + uint64_t rangeStart, + void *user_data) + { + auto *data = static_cast(user_data); + return data->bodyCb ? data->bodyCb(*body, rangeStart) : AWS_OP_SUCCESS; + } + + static int s_onBodyEx( + struct aws_s3_meta_request * /*meta*/, + const ByteCursor *body, + const struct aws_s3_meta_request_receive_body_extra_info info, + void *user_data) + { + auto *data = static_cast(user_data); + if (!data->bodyCbEx) + { + return AWS_OP_SUCCESS; + } + // Stack-only, non-owning wrapper over the CRT's borrowed ticket + // (which may be null, ex. HEAD_OBJECT): no heap allocation and no + // reference taken on the per-chunk path. If the callback needs the + // buffer past its return, it calls ticket.Acquire(), which is the + // only path that takes a reference and allocates. + S3BufferTicket ticket(info.ticket); + return data->bodyCbEx(*body, info.range_start, ticket); + } + + static void s_onProgress( + struct aws_s3_meta_request * /*meta*/, + const struct aws_s3_meta_request_progress *progress, + void *user_data) + { + auto *data = static_cast(user_data); + if (data->progressCb) + { + data->progressCb(progress->bytes_transferred, progress->content_length); + } + } + + static void s_onFinish( + struct aws_s3_meta_request * /*meta*/, + const struct aws_s3_meta_request_result *result, + void *user_data) + { + auto *data = static_cast(user_data); + if (data->finishCb) + { + S3MetaRequestResult cppResult{}; + cppResult.errorCode = result->error_code; + cppResult.responseStatus = result->response_status; + cppResult.errorResponseHeaders = s_materializeHeaders(result->error_response_headers); + // Borrowed view over the CRT-owned buffer (freed when this + // callback returns); empty cursor when there is no body. + cppResult.errorResponseBody = result->error_response_body != nullptr + ? ByteCursorFromByteBuf(*result->error_response_body) + : ByteCursor{0, nullptr}; + cppResult.didValidateChecksum = result->did_validate; + switch (result->validation_algorithm) + { + case AWS_SCA_NONE: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::None; + break; + case AWS_SCA_CRC32C: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::Crc32c; + break; + case AWS_SCA_CRC32: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::Crc32; + break; + case AWS_SCA_SHA1: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::Sha1; + break; + case AWS_SCA_SHA256: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::Sha256; + break; + case AWS_SCA_CRC64NVME: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::Crc64Nvme; + break; + case AWS_SCA_SHA512: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::Sha512; + break; + case AWS_SCA_XXHASH64: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::XXHash64; + break; + case AWS_SCA_XXHASH3_64: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::XXHash3_64; + break; + case AWS_SCA_XXHASH3_128: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::XXHash3_128; + break; + default: + cppResult.validationAlgorithm = S3ChecksumAlgorithm::None; + break; + } + data->finishCb(cppResult); + } + } + + static void s_onShutdown(void *user_data) + { + auto *data = static_cast(user_data); + if (data->shutdownCb) + { + data->shutdownCb(); + } + Delete(data, ApiAllocator()); + } + + // CRT client shutdown is async and outlives the S3Client object, + // so the callback bundle is heap-allocated and self-frees in the + // trampoline. + struct S3ClientShutdownCallbackData + { + std::function callback; + }; + + static void s_onClientShutdown(void *user_data) + { + auto *data = static_cast(user_data); + if (data->callback) + { + data->callback(); + } + Delete(data, ApiAllocator()); + } + + S3Client::S3Client(const S3ClientConfig &config) noexcept : m_lastError(AWS_ERROR_SUCCESS) + { + Allocator *allocator = ApiAllocator(); + struct aws_s3_client_config *rawConfig = config.GetUnderlyingHandle(); + + // aws-c-s3 mandates a non-null client_bootstrap; fall back to + // the process-global default when the caller didn't set one. + if (rawConfig->client_bootstrap == nullptr) + { + Io::ClientBootstrap *defaultBootstrap = ApiHandle::GetOrCreateStaticDefaultClientBootstrap(); + if (defaultBootstrap != nullptr) + { + rawConfig->client_bootstrap = defaultBootstrap->GetUnderlyingHandle(); + } + } + + // Materialize the retry strategy now that the event loop + // group is settled. Factory path wins; Default leaves it null + // for the CRT to build its own. aws_s3_client_new acquires + // its own ref, so this binding may drop after construction. + Optional retryStrategy; + if (config.GetRetryStrategyFactory()) + { + retryStrategy = config.GetRetryStrategyFactory()(config); + } + else + { + switch (config.GetRetryStrategyType()) + { + case S3RetryStrategyType::Standard: + retryStrategy = S3RetryStrategy::CreateStandard(); + break; + case S3RetryStrategyType::ExponentialBackoff: + { + // Enum path has no EventLoopGroup handle to hand + // to the factory; source it from the bootstrap. + const auto &opts = config.GetRetryStrategyOptions(); + struct aws_exponential_backoff_retry_options backoffOptions; + AWS_ZERO_STRUCT(backoffOptions); + if (rawConfig->client_bootstrap != nullptr) + { + backoffOptions.el_group = rawConfig->client_bootstrap->event_loop_group; + } + backoffOptions.max_retries = opts.maxRetries; + backoffOptions.backoff_scale_factor_ms = opts.scaleFactorMs; + backoffOptions.max_backoff_secs = opts.maxBackoffSecs; + backoffOptions.jitter_mode = AWS_EXPONENTIAL_BACKOFF_JITTER_FULL; + retryStrategy = + S3RetryStrategy(aws_retry_strategy_new_exponential_backoff(allocator, &backoffOptions)); + break; + } + case S3RetryStrategyType::NoRetry: + retryStrategy = S3RetryStrategy::CreateNoRetry(); + break; + case S3RetryStrategyType::Default: + break; + } + } + if (retryStrategy && retryStrategy->GetUnderlyingHandle() != nullptr) + { + rawConfig->retry_strategy = retryStrategy->GetUnderlyingHandle(); + } + + // Local cursor array; aws_s3_client_new deep-copies it. + const Vector &networkInterfaceNames = config.GetNetworkInterfaceNames(); + Vector networkInterfaceNameCursors; + if (!networkInterfaceNames.empty()) + { + networkInterfaceNameCursors.reserve(networkInterfaceNames.size()); + for (const auto &name : networkInterfaceNames) + { + networkInterfaceNameCursors.push_back(ByteCursorFromString(name)); + } + rawConfig->network_interface_names_array = networkInterfaceNameCursors.data(); + rawConfig->num_network_interface_names = networkInterfaceNameCursors.size(); + } + + // Bundle is freed by s_onClientShutdown, which the CRT + // fires only on successful client creation. + S3ClientShutdownCallbackData *shutdownData = nullptr; + if (config.GetClientShutdownCallback()) + { + shutdownData = New(allocator); + shutdownData->callback = config.GetClientShutdownCallback(); + rawConfig->shutdown_callback = s_onClientShutdown; + rawConfig->shutdown_callback_user_data = shutdownData; + } + + m_client = ScopedResource( + aws_s3_client_new(allocator, rawConfig), aws_s3_client_release); + m_lastError = m_client ? AWS_ERROR_SUCCESS : aws_last_error(); + + // aws_s3_client_new deep-copies what it needs, so clear the fields + // that were pointed at constructor-locals (the retry strategy and + // the network-interface cursor array). Otherwise the caller's + // config would be left holding dangling pointers into freed stack + // storage, which would be read if it were reused to build another + // client. + rawConfig->retry_strategy = nullptr; + rawConfig->network_interface_names_array = nullptr; + rawConfig->num_network_interface_names = 0; + + // On failure the CRT won't invoke the shutdown callback; free + // the bundle here to avoid leaking it. + if (m_client == nullptr && shutdownData != nullptr) + { + Delete(shutdownData, allocator); + } + } + + int S3Client::LastError() const noexcept + { + return m_lastError ? m_lastError : AWS_ERROR_UNKNOWN; + } + + bool S3Client::MakeDefaultSigningConfig( + Auth::AwsSigningConfig &config, + const String ®ion, + const std::shared_ptr &provider) noexcept + { + // aws_s3_init_default_signing_config's precondition aborts + // under DEBUG_BUILD if the provider is null. + if (provider == nullptr) + { + return false; + } + + struct aws_signing_config_aws raw; + AWS_ZERO_STRUCT(raw); + aws_s3_init_default_signing_config(&raw, ByteCursorFromString(region), provider->GetUnderlyingHandle()); + + config.SetSigningAlgorithm(static_cast(raw.algorithm)); + config.SetSignatureType(static_cast(raw.signature_type)); + config.SetRegion(region); + config.SetService(String(reinterpret_cast(raw.service.ptr), raw.service.len)); + config.SetSignedBodyHeader(static_cast(raw.signed_body_header)); + config.SetSignedBodyValue( + String(reinterpret_cast(raw.signed_body_value.ptr), raw.signed_body_value.len)); + config.SetCredentialsProvider(provider); + + // S3 requires single URI encoding and no path normalization + // or requests for keys with reserved characters fail with + // SignatureDoesNotMatch. The C++ ctor defaults both to true. + config.SetUseDoubleUriEncode(false); + config.SetShouldNormalizeUriPath(false); + return true; + } + + std::shared_ptr S3Client::MakeMetaRequest(S3MetaRequestOptions &options) noexcept + { + if (!*this) + { + m_lastError = AWS_ERROR_INVALID_STATE; + return nullptr; + } + + if (options.GetBodyCallback() && options.GetBodyCallbackEx()) + { + m_lastError = AWS_ERROR_INVALID_ARGUMENT; + return nullptr; + } + + // Surface a validation error deferred by a Set* setter (ex. an + // invalid endpoint URI) rather than issuing a malformed request. + if (options.GetLastError() != AWS_ERROR_SUCCESS) + { + m_lastError = options.GetLastError(); + return nullptr; + } + + Allocator *allocator = ApiAllocator(); + auto *callbackData = New(allocator); + // Copy (not move) the callbacks out: the getters expose them as + // const refs so the options object cannot be left in a half-moved + // state, and copying a std::function is cheap. + callbackData->bodyCb = options.GetBodyCallback(); + callbackData->bodyCbEx = options.GetBodyCallbackEx(); + callbackData->headersCb = options.GetHeadersCallback(); + callbackData->progressCb = options.GetProgressCallback(); + callbackData->finishCb = options.GetFinishCallback(); + callbackData->shutdownCb = options.GetShutdownCallback(); + + auto wrapper = Aws::Crt::MakeShared(allocator); + callbackData->wrapper = wrapper; + + struct aws_s3_meta_request_options *rawOptions = options.GetUnderlyingHandle(); + rawOptions->user_data = callbackData; + rawOptions->headers_callback = s_onHeaders; + rawOptions->body_callback = callbackData->bodyCb ? s_onBody : nullptr; + rawOptions->body_callback_ex = callbackData->bodyCbEx ? s_onBodyEx : nullptr; + rawOptions->progress_callback = s_onProgress; + rawOptions->finish_callback = s_onFinish; + rawOptions->shutdown_callback = s_onShutdown; + + struct aws_s3_meta_request *rawHandle = aws_s3_client_make_meta_request(m_client.get(), rawOptions); + if (rawHandle == nullptr) + { + m_lastError = aws_last_error(); + Delete(callbackData, allocator); + return nullptr; + } + wrapper->SetUnderlyingHandle(rawHandle); + return wrapper; + } + + S3MetaRequest::S3MetaRequest() noexcept : m_lastError(AWS_ERROR_SUCCESS) {} + + void S3MetaRequest::SetUnderlyingHandle(struct aws_s3_meta_request *handle) noexcept + { + m_metaRequest = ScopedResource(handle, aws_s3_meta_request_release); + } + + void S3MetaRequest::Cancel() noexcept + { + aws_s3_meta_request_cancel(m_metaRequest.get()); + } + + void S3MetaRequest::IncrementReadWindow(uint64_t bytes) noexcept + { + aws_s3_meta_request_increment_read_window(m_metaRequest.get(), bytes); + } + + // Bridges the aws_future_void returned by aws_s3_meta_request_write to a + // std::promise. Heap-allocated so it outlives Write(); freed in the + // completion trampoline. + struct S3MetaRequestWriteData + { + std::promise WritePromise; + struct aws_future_void *WriteFuture = nullptr; + + static void OnWriteComplete(void *userData) + { + auto writeData = static_cast(userData); + writeData->WritePromise.set_value(aws_future_void_get_error(writeData->WriteFuture)); + aws_future_void_release(writeData->WriteFuture); + Aws::Crt::Delete(writeData, ApiAllocator()); + } + }; + + std::future S3MetaRequest::Write(ByteCursor data, bool eof) noexcept + { + auto *writeData = Aws::Crt::New(ApiAllocator()); + if (writeData == nullptr) + { + std::promise failed; + failed.set_value(aws_last_error()); + return failed.get_future(); + } + + auto future = writeData->WritePromise.get_future(); + writeData->WriteFuture = aws_s3_meta_request_write(m_metaRequest.get(), data, eof); + aws_future_void_register_callback( + writeData->WriteFuture, S3MetaRequestWriteData::OnWriteComplete, writeData); + return future; + } + + int S3MetaRequest::LastError() const noexcept + { + return m_lastError ? m_lastError : AWS_ERROR_UNKNOWN; + } + + } // namespace S3 + } // namespace Crt +} // namespace Aws diff --git a/source/s3/S3BufferTicket.cpp b/source/s3/S3BufferTicket.cpp new file mode 100644 index 000000000..1a9edd4cf --- /dev/null +++ b/source/s3/S3BufferTicket.cpp @@ -0,0 +1,57 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include + +#include + +#include + +namespace Aws +{ + namespace Crt + { + namespace S3 + { + // Non-owning: wraps a borrowed CRT ticket handle. The wrapper never + // releases the handle itself - the reference (if any) is owned by + // the caller. The stack ticket handed to BodyCallbackEx uses this + // directly; Acquire() takes a reference and puts the matching + // release in its shared_ptr deleter. + S3BufferTicket::S3BufferTicket(struct aws_s3_buffer_ticket *ticket) noexcept : m_ticket(ticket) {} + + std::shared_ptr S3BufferTicket::Acquire() noexcept + { + if (m_ticket == nullptr) + { + return nullptr; + } + // Take a reference for the caller, then hand back a heap wrapper + // whose deleter both releases that reference and frees the + // wrapper through the CRT allocator. This is the only path that + // allocates: the per-chunk callback ticket stays on the stack. + aws_s3_buffer_ticket_acquire(m_ticket); + struct aws_s3_buffer_ticket *acquired = m_ticket; + return std::shared_ptr( + Aws::Crt::New(ApiAllocator(), acquired), + [acquired](S3BufferTicket *p) + { + aws_s3_buffer_ticket_release(acquired); + Delete(p, ApiAllocator()); + }); + } + + ByteCursor S3BufferTicket::Claim() noexcept + { + if (m_ticket == nullptr) + { + return ByteCursor{0, nullptr}; + } + struct aws_byte_buf buffer = aws_s3_buffer_ticket_claim(m_ticket); + return ByteCursorFromByteBuf(buffer); + } + + } // namespace S3 + } // namespace Crt +} // namespace Aws