From f392202c28217eada46a8634b7f6eaea944d40c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Silva=20Gouv=C3=AAa?= <64869705+gouveags@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:30:29 -0300 Subject: [PATCH 1/5] Implement modeled gzip request compression across Python protocols --- CHANGELOG.md | 4 + .../tests/test_awsjson10_protocol.py | 2 - .../tests/test_awsjson11_protocol.py | 2 - .../aws-query/tests/test_awsquery_protocol.py | 2 - .../ec2-query/tests/test_ec2query_protocol.py | 2 - .../tests/test_restjson_protocol.py | 2 - .../rest-xml/tests/test_restxml_protocol.py | 2 - .../rpcv2-cbor/src/rpcv2cbor/config.py | 26 +++ .../codegen/AwsJson10ProtocolGenerator.java | 3 - .../codegen/AwsJson11ProtocolGenerator.java | 3 - .../codegen/AwsQueryProtocolGenerator.java | 4 - .../codegen/Ec2QueryProtocolGenerator.java | 4 - .../aws/codegen/RestXmlProtocolGenerator.java | 4 - .../RequestCompressionIntegration.java | 49 ++++ .../RestJsonProtocolGenerator.java | 5 - ...hon.codegen.integrations.PythonIntegration | 2 + ...ture-f159d224eecb4baf90f101c7dfaebdb3.json | 4 + packages/smithy-aws-core/README.md | 30 +++ .../src/smithy_aws_core/aio/protocols.py | 41 ++-- .../src/smithy_aws_core/config/aws_config.py | 39 ++++ .../src/smithy_aws_core/config/resolvers.py | 28 +++ .../src/smithy_aws_core/config/validators.py | 26 +++ .../tests/unit/aio/test_protocols.py | 220 +++++++++++++++++- .../tests/unit/config/test_resolver.py | 100 ++++++++ ...gfix-4cef458d2ac549dab2645848ab26eb6b.json | 4 + ...ture-76b67ff1b28948f19f2f4ffb34dd75e3.json | 4 + .../smithy-core/src/smithy_core/aio/utils.py | 11 +- .../smithy-core/src/smithy_core/traits.py | 13 ++ .../smithy-core/tests/unit/aio/test_utils.py | 38 +++ .../tests/unit/test_schema_compact.py | 18 +- .../smithy-core/tests/unit/test_traits.py | 7 + ...gfix-2b89be327ad14f599285697d48eb61e3.json | 4 + ...ture-dc6d4536b20e4253a86771e28a53dfd9.json | 4 + .../src/smithy_http/aio/aiohttp.py | 8 +- .../src/smithy_http/aio/protocols.py | 12 +- .../src/smithy_http/compression.py | 174 ++++++++++++++ .../src/smithy_http/serializers.py | 24 ++ .../tests/unit/aio/test_aiohttp.py | 82 +++++++ .../tests/unit/aio/test_protocols.py | 98 ++++++++ .../tests/unit/test_compression.py | 195 ++++++++++++++++ 40 files changed, 1235 insertions(+), 65 deletions(-) create mode 100644 codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RequestCompressionIntegration.java create mode 100644 packages/smithy-aws-core/.changes/next-release/smithy-aws-core-feature-f159d224eecb4baf90f101c7dfaebdb3.json create mode 100644 packages/smithy-core/.changes/next-release/smithy-core-bugfix-4cef458d2ac549dab2645848ab26eb6b.json create mode 100644 packages/smithy-core/.changes/next-release/smithy-core-feature-76b67ff1b28948f19f2f4ffb34dd75e3.json create mode 100644 packages/smithy-http/.changes/next-release/smithy-http-bugfix-2b89be327ad14f599285697d48eb61e3.json create mode 100644 packages/smithy-http/.changes/next-release/smithy-http-feature-dc6d4536b20e4253a86771e28a53dfd9.json create mode 100644 packages/smithy-http/src/smithy_http/compression.py create mode 100644 packages/smithy-http/tests/unit/test_compression.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 30cba9198..a43a757b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## Unreleased +### Features + +* Generated HTTP clients expose request compression controls for operations marked with the `requestCompression` trait. + ## v0.6.0 ### Breaking Changes diff --git a/codegen-output/aws-json-1-0/tests/test_awsjson10_protocol.py b/codegen-output/aws-json-1-0/tests/test_awsjson10_protocol.py index 6a4c1103a..5edae1cb5 100644 --- a/codegen-output/aws-json-1-0/tests/test_awsjson10_protocol.py +++ b/codegen-output/aws-json-1-0/tests/test_awsjson10_protocol.py @@ -3106,7 +3106,6 @@ async def test_aws_json10_client_error_corrects_with_default_values_when_server_ assert deep_equal(actual, expected) -@mark.xfail() async def test_sdk_applied_content_encoding_aws_json1_0_request_put_with_content_encoding() -> ( None ): @@ -3180,7 +3179,6 @@ async def test_sdk_applied_content_encoding_aws_json1_0_request_put_with_content ) -@mark.xfail() async def test_sdk_appends_gzip_and_ignores_http_provided_encoding_aws_json1_0_request_put_with_content_encoding() -> ( None ): diff --git a/codegen-output/aws-json-1-1/tests/test_awsjson11_protocol.py b/codegen-output/aws-json-1-1/tests/test_awsjson11_protocol.py index c544df3fa..3d6c721f9 100644 --- a/codegen-output/aws-json-1-1/tests/test_awsjson11_protocol.py +++ b/codegen-output/aws-json-1-1/tests/test_awsjson11_protocol.py @@ -5746,7 +5746,6 @@ async def test_put_and_get_inline_documents_input_response_put_and_get_inline_do assert deep_equal(actual, expected) -@mark.xfail() async def test_sdk_applied_content_encoding_aws_json1_1_request_put_with_content_encoding() -> ( None ): @@ -5820,7 +5819,6 @@ async def test_sdk_applied_content_encoding_aws_json1_1_request_put_with_content ) -@mark.xfail() async def test_sdk_appends_gzip_and_ignores_http_provided_encoding_aws_json1_1_request_put_with_content_encoding() -> ( None ): diff --git a/codegen-output/aws-query/tests/test_awsquery_protocol.py b/codegen-output/aws-query/tests/test_awsquery_protocol.py index 963b63e3e..f9efe7dee 100644 --- a/codegen-output/aws-query/tests/test_awsquery_protocol.py +++ b/codegen-output/aws-query/tests/test_awsquery_protocol.py @@ -1104,7 +1104,6 @@ async def test_query_no_input_and_output_response_no_input_and_output() -> None: assert deep_equal(actual, expected) -@mark.xfail() async def test_sdk_applied_content_encoding_aws_query_request_put_with_content_encoding() -> ( None ): @@ -1178,7 +1177,6 @@ async def test_sdk_applied_content_encoding_aws_query_request_put_with_content_e ) -@mark.xfail() async def test_sdk_appends_gzip_and_ignores_http_provided_encoding_aws_query_request_put_with_content_encoding() -> ( None ): diff --git a/codegen-output/ec2-query/tests/test_ec2query_protocol.py b/codegen-output/ec2-query/tests/test_ec2query_protocol.py index 2a5b75531..1b85a725b 100644 --- a/codegen-output/ec2-query/tests/test_ec2query_protocol.py +++ b/codegen-output/ec2-query/tests/test_ec2query_protocol.py @@ -839,7 +839,6 @@ async def test_ec2_query_no_input_and_output_response_no_input_and_output() -> N assert deep_equal(actual, expected) -@mark.xfail() async def test_sdk_applied_content_encoding_ec2_query_request_put_with_content_encoding() -> ( None ): @@ -913,7 +912,6 @@ async def test_sdk_applied_content_encoding_ec2_query_request_put_with_content_e ) -@mark.xfail() async def test_sdk_appends_gzip_and_ignores_http_provided_encoding_ec2_query_request_put_with_content_encoding() -> ( None ): diff --git a/codegen-output/rest-json-1/tests/test_restjson_protocol.py b/codegen-output/rest-json-1/tests/test_restjson_protocol.py index 732e0ed52..44f064ad8 100644 --- a/codegen-output/rest-json-1/tests/test_restjson_protocol.py +++ b/codegen-output/rest-json-1/tests/test_restjson_protocol.py @@ -10786,7 +10786,6 @@ async def test_post_union_with_json_name_response3_response_post_union_with_json assert deep_equal(actual, expected) -@mark.xfail() async def test_sdk_applied_content_encoding_rest_json1_request_put_with_content_encoding() -> ( None ): @@ -10860,7 +10859,6 @@ async def test_sdk_applied_content_encoding_rest_json1_request_put_with_content_ ) -@mark.xfail() async def test_sdk_appended_gzip_after_provided_encoding_rest_json1_request_put_with_content_encoding() -> ( None ): diff --git a/codegen-output/rest-xml/tests/test_restxml_protocol.py b/codegen-output/rest-xml/tests/test_restxml_protocol.py index fbc3216d3..142593887 100644 --- a/codegen-output/rest-xml/tests/test_restxml_protocol.py +++ b/codegen-output/rest-xml/tests/test_restxml_protocol.py @@ -5714,7 +5714,6 @@ async def test_rest_xml_serializes_empty_string_request_omits_null_serializes_em ) -@mark.xfail() async def test_sdk_applied_content_encoding_rest_xml_request_put_with_content_encoding() -> ( None ): @@ -5788,7 +5787,6 @@ async def test_sdk_applied_content_encoding_rest_xml_request_put_with_content_en ) -@mark.xfail() async def test_sdk_appended_gzip_after_provided_encoding_rest_xml_request_put_with_content_encoding() -> ( None ): diff --git a/codegen-output/rpcv2-cbor/src/rpcv2cbor/config.py b/codegen-output/rpcv2-cbor/src/rpcv2cbor/config.py index 41048cc80..1aae7272f 100644 --- a/codegen-output/rpcv2-cbor/src/rpcv2cbor/config.py +++ b/codegen-output/rpcv2-cbor/src/rpcv2cbor/config.py @@ -91,6 +91,9 @@ class Config: auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] """A map of auth scheme ids to auth schemes.""" + disable_request_compression: bool + """Disable compression on operations that advertise support.""" + endpoint_resolver: EndpointResolver """ The endpoint resolver used to resolve the final endpoint per-operation @@ -122,6 +125,15 @@ class Config: the service endpoint. """ + request_compression_encoding: str | None + """Explicitly request gzip regardless of size; unsupported operations fail.""" + + request_min_compression_size_bytes: int + """ + Minimum buffered request size for automatic compression; zero compresses + all. + """ + retry_strategy: RetryStrategy | RetryStrategyOptions | None """ The retry strategy or options for configuring retry behavior. Can be @@ -150,6 +162,7 @@ def __init__( *, auth_scheme_resolver: AuthSchemeResolver | None = None, auth_schemes: dict[ShapeID, AuthScheme[Any, Any, Any, Any]] | None = None, + disable_request_compression: bool | None = None, endpoint_resolver: EndpointResolver | None = None, endpoint_uri: str | URI | None = None, http_request_config: HTTPRequestConfiguration | None = None, @@ -158,6 +171,8 @@ def __init__( | ProtocolConstructor[ClientProtocol[Any, Any]] | None = None, region: str | None = None, + request_compression_encoding: str | None = None, + request_min_compression_size_bytes: int | None = None, retry_strategy: RetryStrategy | RetryStrategyOptions | None = None, sdk_ua_app_id: str | None = None, transport: ClientTransport[Any, Any] | None = None, @@ -165,6 +180,11 @@ def __init__( ): self.auth_scheme_resolver = auth_scheme_resolver or DefaultAuthResolver() self.auth_schemes = auth_schemes or {} + self.disable_request_compression = ( + disable_request_compression + if disable_request_compression is not None + else False + ) self.endpoint_resolver = endpoint_resolver or StandardRegionalEndpointsResolver( endpoint_prefix="RpcV2Protocol" ) @@ -177,6 +197,12 @@ def __init__( "ClientProtocol[Any, Any]", protocol ) or RpcV2CborClientProtocol(_PROTOCOL_SETTINGS) self.region = region + self.request_compression_encoding = request_compression_encoding + self.request_min_compression_size_bytes = ( + request_min_compression_size_bytes + if request_min_compression_size_bytes is not None + else 10240 + ) self.retry_strategy = retry_strategy self.sdk_ua_app_id = sdk_ua_app_id self.transport = transport or AIOHTTPClient() diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsJson10ProtocolGenerator.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsJson10ProtocolGenerator.java index f9953dd46..ce2ab2304 100644 --- a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsJson10ProtocolGenerator.java +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsJson10ProtocolGenerator.java @@ -19,9 +19,6 @@ @SmithyInternalApi public final class AwsJson10ProtocolGenerator implements ProtocolGenerator { private static final Set TESTS_TO_SKIP = Set.of( - // TODO: support the request compression trait. - "SDKAppliedContentEncoding_awsJson1_0", - "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsJson1_0", // TODO: fix default value behavior for JSON RPC. "AwsJson10ClientPopulatesDefaultValuesInInput", diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsJson11ProtocolGenerator.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsJson11ProtocolGenerator.java index c3ad4946f..e42585a4b 100644 --- a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsJson11ProtocolGenerator.java +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsJson11ProtocolGenerator.java @@ -19,9 +19,6 @@ @SmithyInternalApi public final class AwsJson11ProtocolGenerator implements ProtocolGenerator { private static final Set TESTS_TO_SKIP = Set.of( - // TODO: support the request compression trait. - "SDKAppliedContentEncoding_awsJson1_1", - "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsJson1_1", // TODO: support the endpoint trait. "AwsJson11EndpointTrait", diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsQueryProtocolGenerator.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsQueryProtocolGenerator.java index 862deb912..289277dfe 100644 --- a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsQueryProtocolGenerator.java +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsQueryProtocolGenerator.java @@ -18,10 +18,6 @@ @SmithyInternalApi public final class AwsQueryProtocolGenerator implements ProtocolGenerator { private static final Set TESTS_TO_SKIP = Set.of( - // TODO: support the request compression trait - // https://smithy.io/2.0/spec/behavior-traits.html#smithy-api-requestcompression-trait - "SDKAppliedContentEncoding_awsQuery", - "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsQuery", // TODO: support idempotency token autofill "QueryProtocolIdempotencyTokenAutoFill", diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/Ec2QueryProtocolGenerator.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/Ec2QueryProtocolGenerator.java index c10aaadff..d87315386 100644 --- a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/Ec2QueryProtocolGenerator.java +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/Ec2QueryProtocolGenerator.java @@ -18,10 +18,6 @@ @SmithyInternalApi public final class Ec2QueryProtocolGenerator implements ProtocolGenerator { private static final Set TESTS_TO_SKIP = Set.of( - // TODO: support the request compression trait - // https://smithy.io/2.0/spec/behavior-traits.html#smithy-api-requestcompression-trait - "SDKAppliedContentEncoding_ec2Query", - "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_ec2Query", // TODO: support idempotency token autofill "Ec2ProtocolIdempotencyTokenAutoFill", diff --git a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/RestXmlProtocolGenerator.java b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/RestXmlProtocolGenerator.java index fc714904c..9a023020d 100644 --- a/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/RestXmlProtocolGenerator.java +++ b/codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/RestXmlProtocolGenerator.java @@ -20,10 +20,6 @@ @SmithyInternalApi public final class RestXmlProtocolGenerator implements ProtocolGenerator { private static final Set TESTS_TO_SKIP = Set.of( - // TODO: support the request compression trait - // https://smithy.io/2.0/spec/behavior-traits.html#smithy-api-requestcompression-trait - "SDKAppliedContentEncoding_restXml", - "SDKAppendedGzipAfterProvidedEncoding_restXml", // TODO: support idempotency token autofill "QueryIdempotencyTokenAutoFill", diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RequestCompressionIntegration.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RequestCompressionIntegration.java new file mode 100644 index 000000000..5d81ea54a --- /dev/null +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RequestCompressionIntegration.java @@ -0,0 +1,49 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ +package software.amazon.smithy.python.codegen.integrations; + +import java.util.List; +import software.amazon.smithy.aws.traits.ServiceTrait; +import software.amazon.smithy.codegen.core.Symbol; +import software.amazon.smithy.python.codegen.ConfigProperty; +import software.amazon.smithy.python.codegen.GenerationContext; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** Adds modeled request-compression controls to HTTP clients. */ +@SmithyInternalApi +public final class RequestCompressionIntegration implements PythonIntegration { + @Override + public List getClientPlugins(GenerationContext context) { + if (!context.applicationProtocol().isHttpProtocol()) { + return List.of(); + } + return List.of(RuntimeClientPlugin.builder() + // AWS clients inherit the controls and resolvers from AsyncAwsConfig. + .servicePredicate((model, service) -> !service.hasTrait(ServiceTrait.class)) + .addConfigProperty(ConfigProperty.builder() + .name("disable_request_compression") + .type(Symbol.builder().name("bool").build()) + .nullable(false) + .documentation("Disable compression on operations that advertise support.") + .initialize(w -> w.write("self.disable_request_compression = " + + "disable_request_compression if disable_request_compression is not None else False")) + .build()) + .addConfigProperty(ConfigProperty.builder() + .name("request_min_compression_size_bytes") + .type(Symbol.builder().name("int").build()) + .nullable(false) + .documentation("Minimum buffered request size for automatic compression; zero compresses all.") + .initialize(w -> w.write("self.request_min_compression_size_bytes = " + + "request_min_compression_size_bytes if request_min_compression_size_bytes is not None " + + "else 10240")) + .build()) + .addConfigProperty(ConfigProperty.builder() + .name("request_compression_encoding") + .type(Symbol.builder().name("str").build()) + .documentation("Explicitly request gzip regardless of size; unsupported operations fail.") + .build()) + .build()); + } +} diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RestJsonProtocolGenerator.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RestJsonProtocolGenerator.java index 2701b6a6d..f9f5ab74d 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RestJsonProtocolGenerator.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RestJsonProtocolGenerator.java @@ -41,11 +41,6 @@ public class RestJsonProtocolGenerator implements ProtocolGenerator { "RestJsonEndpointTraitWithHostLabel", "RestJsonEndpointTrait", - // TODO: support the request compression trait - // https://smithy.io/2.0/spec/behavior-traits.html#smithy-api-requestcompression-trait - "SDKAppliedContentEncoding_restJson1", - "SDKAppendedGzipAfterProvidedEncoding_restJson1", - // TODO: These tests require a payload even when the httpPayload member is null. Should it? "RestJsonHttpWithHeadersButNoPayload", "RestJsonHttpWithEmptyStructurePayload", diff --git a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration index ccdf63695..a3c5704d8 100644 --- a/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration +++ b/codegen/core/src/main/resources/META-INF/services/software.amazon.smithy.python.codegen.integrations.PythonIntegration @@ -8,3 +8,5 @@ software.amazon.smithy.python.codegen.integrations.Rpcv2CborIntegration software.amazon.smithy.python.codegen.integrations.HttpApiKeyAuth software.amazon.smithy.python.codegen.integrations.UserAgentIntegration software.amazon.smithy.python.codegen.integrations.EndpointBddIntegration + +software.amazon.smithy.python.codegen.integrations.RequestCompressionIntegration diff --git a/packages/smithy-aws-core/.changes/next-release/smithy-aws-core-feature-f159d224eecb4baf90f101c7dfaebdb3.json b/packages/smithy-aws-core/.changes/next-release/smithy-aws-core-feature-f159d224eecb4baf90f101c7dfaebdb3.json new file mode 100644 index 000000000..cd4d0a74f --- /dev/null +++ b/packages/smithy-aws-core/.changes/next-release/smithy-aws-core-feature-f159d224eecb4baf90f101c7dfaebdb3.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Resolve request compression settings from client overrides, environment variables, and shared AWS configuration, and apply compression across AWS HTTP protocols." +} \ No newline at end of file diff --git a/packages/smithy-aws-core/README.md b/packages/smithy-aws-core/README.md index 150feaef1..07294e30a 100644 --- a/packages/smithy-aws-core/README.md +++ b/packages/smithy-aws-core/README.md @@ -2,3 +2,33 @@ This is the core package that provides AWS specific interfaces for both client and server tooling generated by Smithy. + + +## Request compression + +HTTP operations with Smithy's `requestCompression` trait use gzip automatically +for buffered payloads of at least 10,240 bytes. Eligible streaming blobs are compressed +without checking their size. Event streams and streams requiring a known length +are excluded. Operations without the trait keep their existing behavior; enabling +compression requires support from the service endpoint. + +Seekable input streams remain seekable after compression. SigV4 signing can hash +and rewind them without buffering the entire compressed payload; rewinding +recompresses data as needed. The aiohttp transport checks for an empty body with +a one-byte probe and restores the compressed position instead of scanning to EOF. +Inherently non-seekable streams remain non-seekable, +and the payload-hashing SigV4 signer may buffer them as it does without compression. + +AWS client configuration accepts these settings: + +- `disable_request_compression=True` disables automatic compression. +- `request_min_compression_size_bytes=0` compresses every eligible buffered request. +- `request_compression_encoding="gzip"` explicitly requests gzip regardless of size. + An unsupported operation raises an error, as does combining this option with + `disable_request_compression=True`. + +The standard settings also resolve from `AWS_DISABLE_REQUEST_COMPRESSION` and +`AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES`, or the corresponding lowercase keys in +the shared AWS configuration file. Explicit client settings take precedence. +The explicit encoding option is a client setting only. Zstd is not currently an +encoding supported by Smithy's request-compression trait. diff --git a/packages/smithy-aws-core/src/smithy_aws_core/aio/protocols.py b/packages/smithy-aws-core/src/smithy_aws_core/aio/protocols.py index 77338706a..be8f0350a 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/aio/protocols.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/aio/protocols.py @@ -42,6 +42,7 @@ HttpBindingClientProtocol, HttpClientProtocol, ) +from smithy_http.compression import RequestCompression from smithy_http.deserializers import HTTPResponseDeserializer from .._private.query._xml import parse_xml_root @@ -462,21 +463,21 @@ def serialize_request[ endpoint: URI, context: TypedProperties, ) -> HTTPRequest: + compression = RequestCompression.from_operation(operation, context) payload = self.payload_codec.serialize(shape=input) + fields = tuples_to_fields( + [ + ("content-type", self.content_type), + ("content-length", str(len(payload))), + ("x-amz-target", f"{self._service_name}.{operation.schema.id.name}"), + ] + ) + body = compression.compress(payload, fields) return _HTTPRequest( method="POST", destination=_URI(host="", path="/"), - fields=tuples_to_fields( - [ - ("content-type", self.content_type), - ("content-length", str(len(payload))), - ( - "x-amz-target", - f"{self._service_name}.{operation.schema.id.name}", - ), - ] - ), - body=AsyncBytesReader(payload), + fields=fields, + body=AsyncBytesReader(body), ) async def deserialize_response[ @@ -708,6 +709,7 @@ def serialize_request[ endpoint: URI, context: TypedProperties, ) -> HTTPRequest: + compression = RequestCompression.from_operation(operation, context) sink = BytesIO() params: list[tuple[str, str]] = [] serializer = self._create_serializer( @@ -719,17 +721,18 @@ def serialize_request[ serializer.flush() content_length = sink.tell() sink.seek(0) - body = AsyncBytesReader(sink) + fields = tuples_to_fields( + [ + ("content-type", self.content_type), + ("content-length", str(content_length)), + ] + ) + body = compression.compress(sink, fields) return _HTTPRequest( method="POST", destination=_URI(host="", path="/"), - fields=tuples_to_fields( - [ - ("content-type", self.content_type), - ("content-length", str(content_length)), - ] - ), - body=body, + fields=fields, + body=AsyncBytesReader(body), ) async def deserialize_response[ diff --git a/packages/smithy-aws-core/src/smithy_aws_core/config/aws_config.py b/packages/smithy-aws-core/src/smithy_aws_core/config/aws_config.py index f6ee2c7bd..88b0e76d2 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/config/aws_config.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/config/aws_config.py @@ -24,17 +24,22 @@ from .exceptions import ConfigError, ConfigValidationError from .filesystem import FileSystem from .resolvers import ( + resolve_disable_request_compression, resolve_endpoint_uri, resolve_max_attempts, resolve_region, + resolve_request_min_compression_size_bytes, resolve_retry_mode, resolve_sdk_ua_app_id, ) from .types import UNSET, ConfigSource, FieldSpec, Resolved from .validators import ( + validate_disable_request_compression, validate_max_attempts, validate_profile, validate_region, + validate_request_compression_encoding, + validate_request_min_compression_size_bytes, validate_retry_mode, ) @@ -47,6 +52,9 @@ class AwsConfigOverrides(TypedDict, total=False): region: str | None retry_mode: RetryStrategyType | None max_attempts: int | None + disable_request_compression: bool + request_min_compression_size_bytes: int + request_compression_encoding: str | None endpoint_uri: "str | URI | None" aws_access_key_id: str | None aws_secret_access_key: str | None @@ -88,6 +96,15 @@ class AsyncAwsConfig: """The maximum number of attempts to make per request, including the initial attempt. Must be an integer of at least 1.""" + disable_request_compression: bool = False + """Disable automatic compression on operations that advertise support.""" + + request_min_compression_size_bytes: int = 10240 + """Minimum buffered request size for automatic compression; zero compresses all.""" + + request_compression_encoding: str | None = None + """Explicitly request gzip regardless of size; unsupported operations fail.""" + endpoint_uri: "str | URI | None" = None """A static URI to route requests to.""" @@ -163,6 +180,20 @@ class AsyncAwsConfig: resolver=resolve_max_attempts, validator=validate_max_attempts, ), + "disable_request_compression": FieldSpec( + default=False, + resolver=resolve_disable_request_compression, + validator=validate_disable_request_compression, + ), + "request_min_compression_size_bytes": FieldSpec( + default=10240, + resolver=resolve_request_min_compression_size_bytes, + validator=validate_request_min_compression_size_bytes, + ), + "request_compression_encoding": FieldSpec( + default=None, + validator=validate_request_compression_encoding, + ), "endpoint_uri": FieldSpec( default=None, resolver=resolve_endpoint_uri, @@ -352,6 +383,14 @@ async def _resolve_fields(self, overrides: dict[str, Any]) -> None: if spec.validator is not None: spec.validator(getattr(self, field_name)) + if ( + self.disable_request_compression + and self.request_compression_encoding is not None + ): + raise ConfigValidationError( + "request_compression_encoding conflicts with disable_request_compression" + ) + def _apply_default(self, field_name: str, spec: FieldSpec) -> None: """Apply the default value for a field.""" if spec.default_factory is not None: diff --git a/packages/smithy-aws-core/src/smithy_aws_core/config/resolvers.py b/packages/smithy-aws-core/src/smithy_aws_core/config/resolvers.py index 31b363876..bdd57a76e 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/config/resolvers.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/config/resolvers.py @@ -193,3 +193,31 @@ async def __call__(self, ctx: SharedConfigContext) -> Resolved[str | None]: return Resolved(value=value, source=ConfigSource.PROFILE) return Resolved(value=UNSET, source=ConfigSource.DEFAULT) # type: ignore[arg-type] + + +async def resolve_disable_request_compression( + ctx: SharedConfigContext, +) -> Resolved[bool]: + """Resolve the standard request compression switch.""" + result = await _resolve_str( + ctx, + env_vars=("AWS_DISABLE_REQUEST_COMPRESSION",), + profile_keys=("disable_request_compression",), + ) + if result.value is UNSET: + return Resolved(value=UNSET, source=result.source) # type: ignore[arg-type] + value = str(result.value).lower() + if value not in ("true", "false"): + raise ConfigValidationError("disable_request_compression must be true or false") + return Resolved(value=value == "true", source=result.source) + + +async def resolve_request_min_compression_size_bytes( + ctx: SharedConfigContext, +) -> Resolved[int | None]: + """Resolve the standard buffered request compression threshold.""" + return await _resolve_int( + ctx, + env_vars=("AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES",), + profile_keys=("request_min_compression_size_bytes",), + ) diff --git a/packages/smithy-aws-core/src/smithy_aws_core/config/validators.py b/packages/smithy-aws-core/src/smithy_aws_core/config/validators.py index 376736a26..326f28072 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/config/validators.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/config/validators.py @@ -103,3 +103,29 @@ def validate_profile( raise ProfileNotFoundError( f"Profile {profile_name!r} from {source_str} was not found in config file." ) + + +def validate_disable_request_compression(value: object) -> None: + """Require an explicit boolean for the compression disable switch.""" + if not isinstance(value, bool): + raise ConfigValidationError("disable_request_compression must be a boolean") + + +def validate_request_min_compression_size_bytes(value: object) -> None: + """Validate the standard inclusive compression threshold range.""" + if ( + not isinstance(value, int) + or isinstance(value, bool) + or not 0 <= value <= 10485760 + ): + raise ConfigValidationError( + "request_min_compression_size_bytes must be an integer between 0 and 10485760" + ) + + +def validate_request_compression_encoding(value: object) -> None: + """Only the gzip codec is currently modeled by Smithy.""" + if value not in (None, "gzip"): + raise ConfigValidationError( + "request_compression_encoding must be None or 'gzip'" + ) diff --git a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py index 6792943f1..84928d0ea 100644 --- a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py +++ b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py @@ -1,35 +1,53 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +import gzip from dataclasses import dataclass +from hashlib import sha256 +from io import BytesIO +from types import SimpleNamespace from typing import Any, cast from unittest.mock import Mock, patch from xml.etree import ElementTree import pytest +from aws_sdk_signers import AsyncSigV4Signer, AWSRequest, SigV4SigningProperties from ijson.common import IncompleteJSONError # type: ignore[reportMissingTypeStubs] from smithy_aws_core.aio.protocols import ( AWSErrorIdentifier, AwsJson11ClientProtocol, AWSJSONDocument, AwsQueryClientProtocol, - ProtocolSettings, RestXmlClientProtocol, ) +from smithy_aws_core.auth import SigV4AuthScheme +from smithy_aws_core.auth.sigv4 import SigV4Signer +from smithy_aws_core.identity import AWSCredentialsIdentity +from smithy_aws_core.identity.static import StaticCredentialsResolver from smithy_aws_event_stream.events import EventMessage from smithy_core import URI as _URI +from smithy_core.aio.client import ClientCall, RequestPipeline +from smithy_core.aio.interfaces import ProtocolSettings +from smithy_core.aio.retries import StandardRetryStrategy +from smithy_core.aio.types import AsyncBytesReader +from smithy_core.aio.utils import close +from smithy_core.auth import DefaultAuthResolver from smithy_core.deserializers import ShapeDeserializer from smithy_core.documents import TypeRegistry +from smithy_core.endpoints import Endpoint from smithy_core.exceptions import CallError, DiscriminatorError, ModeledError +from smithy_core.interceptors import InterceptorChain from smithy_core.interfaces import URI from smithy_core.prelude import INTEGER, STRING +from smithy_core.retries import ExponentialRetryBackoffStrategy from smithy_core.schemas import APIOperation, Schema from smithy_core.serializers import ShapeSerializer from smithy_core.shapes import ShapeID, ShapeType -from smithy_core.traits import Trait +from smithy_core.traits import RequestCompressionTrait, Trait from smithy_core.types import TypedProperties from smithy_http import Fields, tuples_to_fields from smithy_http.aio import HTTPRequest, HTTPResponse +from smithy_http.compression import RequestCompression from smithy_json import JSONSettings @@ -200,6 +218,7 @@ def _mock_operation( operation = Mock(spec=APIOperation) operation.schema = schema operation.error_schemas = error_schemas or [] + operation.input_stream_member = None return cast("APIOperation[Any, Any]", operation) @@ -226,6 +245,203 @@ async def test_aws_json11_serializes_base_request_shape() -> None: assert await request.consume_body_async() == b"{}" +@pytest.mark.parametrize("kind", ["json", "query"]) +@pytest.mark.parametrize("explicit", [None, "gzip"]) +async def test_aws_protocol_compresses_serialized_request( + kind: str, explicit: str | None +) -> None: + protocol = ( + _aws_json11_protocol() + if kind == "json" + else AwsQueryClientProtocol( + ProtocolSettings( + namespace="com.test", + service_target="QueryService", + version="2020-01-08", + ) + ) + ) + schema = Schema( + id=ShapeID("com.test#TestOperation"), + shape_type=ShapeType.OPERATION, + traits=[RequestCompressionTrait({"encodings": ["gzip"]})], + ) + operation = _mock_operation(schema) + request = protocol.serialize_request( + operation=operation, + input=_TestInput(name="example"), + endpoint=_URI(host="example.com"), + context=TypedProperties( + { + "config": SimpleNamespace( + disable_request_compression=False, + request_min_compression_size_bytes=0 + if explicit is None + else 10485760, + request_compression_encoding=explicit, + ) + } + ), + ) + compressed = await request.consume_body_async() + assert request.fields["content-encoding"].as_string() == "gzip" + assert request.fields["content-length"].as_string() == str(len(compressed)) + expected = ( + b'{"name":"example"}' + if kind == "json" + else b"Action=TestOperation&Version=2020-01-08&name=example" + ) + assert gzip.decompress(compressed) == expected + + +async def test_compressed_request_is_signed_and_replayed_by_pipeline() -> None: + canonical_payloads: list[str] = [] + transmitted: list[bytes] = [] + + class RecordingSigner(AsyncSigV4Signer): + async def canonical_request( + self, *, signing_properties: SigV4SigningProperties, request: AWSRequest + ) -> str: + canonical = await super().canonical_request( + signing_properties=signing_properties, request=request + ) + canonical_payloads.append(canonical.splitlines()[-1]) + return canonical + + class EndpointResolver: + async def resolve_endpoint(self, params: Any) -> Endpoint: + return Endpoint(uri=_URI(scheme="https", host="example.com")) + + class RetryTransport: + TIMEOUT_EXCEPTIONS: tuple[type[Exception], ...] = () + + async def send(self, request: Any, **kwargs: Any) -> HTTPResponse: + wire = await request.consume_body_async() + transmitted.append(wire) + assert request.fields["content-encoding"].as_string() == "gzip" + assert "content-encoding" in request.fields["authorization"].as_string() + assert int(request.fields["content-length"].as_string()) == len(wire) + return HTTPResponse( + status=429 if len(transmitted) == 1 else 200, + fields=Fields(), + body=b'{"__type":"ThrottlingException"}' + if len(transmitted) == 1 + else b"{}", + ) + + scheme = SigV4AuthScheme( + service="testservice", signer=cast(SigV4Signer, RecordingSigner()) + ) + config = SimpleNamespace( + region="us-west-2", + aws_access_key_id=None, + aws_secret_access_key=None, + aws_session_token=None, + aws_credentials_identity_resolver=StaticCredentialsResolver( + AWSCredentialsIdentity( + access_key_id="test-access-key", secret_access_key="test-secret-key" + ) + ), + disable_request_compression=False, + request_min_compression_size_bytes=10240, + request_compression_encoding="gzip", + ) + operation = APIOperation( + input=_TestInput, + output=_EmptyOutput, + static_schema=( + ShapeType.OPERATION.value, + "com.test", + "TestOperation", + {"smithy.api#requestCompression": {"encodings": ["gzip"]}}, + _INPUT_SCHEMA, + _EMPTY_OUTPUT_SCHEMA, + (), + ), + error_registry=TypeRegistry({}), + effective_auth_schemes=[scheme.scheme_id], + ) + call = ClientCall( + input=_TestInput(name="example"), + operation=operation, + context=TypedProperties({"config": config}), + interceptor=InterceptorChain([]), + auth_scheme_resolver=DefaultAuthResolver(), + supported_auth_schemes={scheme.scheme_id: scheme}, + endpoint_resolver=EndpointResolver(), + retry_strategy=StandardRetryStrategy( + max_attempts=2, + backoff_strategy=ExponentialRetryBackoffStrategy(backoff_scale_value=0), + ), + ) + output = await RequestPipeline( + protocol=_aws_json11_protocol(), transport=RetryTransport() + )(call) + assert isinstance(output, _EmptyOutput) + assert len(transmitted) == 2 + assert transmitted[0] == transmitted[1] + assert gzip.decompress(transmitted[0]) == b'{"name":"example"}' + assert canonical_payloads == [sha256(transmitted[0]).hexdigest()] * 2 + + +async def test_signed_compressed_seekable_stream_is_not_fully_buffered() -> None: + payload = b"".join( + sha256(index.to_bytes(4, "big")).digest() for index in range(8192) + ) + source = BytesIO(b"prefix" + payload) + source.seek(6) + fields = Fields() + body = RequestCompression(encoding="gzip", minimum_size=0).compress( + source, fields, streaming=True + ) + assert isinstance(body, AsyncBytesReader) + request = HTTPRequest( + destination=_URI(scheme="https", host="example.com"), + method="POST", + fields=fields, + body=body, + ) + signed = await AsyncSigV4Signer().sign( + request=cast(AWSRequest, request), + identity=AWSCredentialsIdentity( + access_key_id="test-key", secret_access_key="test-secret" + ), + properties={ + "region": "us-west-2", + "service": "testservice", + "payload_signing_enabled": True, + "content_checksum_enabled": True, + }, + ) + assert signed.body is body + assert body.seekable() + assert body.tell() == 0 + wire = await body.read() + assert gzip.decompress(wire) == payload + assert signed.fields["X-Amz-Content-SHA256"].as_string() == sha256(wire).hexdigest() + await body.seek(0) + assert await body.read() == wire + await body.seek(17) + assert body.tell() == 17 + assert await body.read(31) == wire[17:48] + assert await body.seek(5, 1) == 53 + assert await body.read(3) == wire[53:56] + assert await body.seek(-19, 2) == len(wire) - 19 + assert await body.read() == wire[-19:] + assert await body.seek(len(wire) + 100) == len(wire) + assert await body.read() == b"" + with pytest.raises(ValueError, match="Negative"): + await body.seek(-1) + with pytest.raises(ValueError, match="whence"): + await body.seek(0, 99) + await body.seek(0) + assert await body.read() == wire + await close(body) + assert source.closed + with pytest.raises(ValueError, match="closed"): + await body.seek(0) + + async def test_aws_json11_resolves_body_error_with_content_type_parameters() -> None: protocol = _aws_json11_protocol() response = HTTPResponse( diff --git a/packages/smithy-aws-core/tests/unit/config/test_resolver.py b/packages/smithy-aws-core/tests/unit/config/test_resolver.py index 04014d94b..8594c4197 100644 --- a/packages/smithy-aws-core/tests/unit/config/test_resolver.py +++ b/packages/smithy-aws-core/tests/unit/config/test_resolver.py @@ -11,6 +11,7 @@ from copy import deepcopy from dataclasses import dataclass from inspect import signature +from typing import Any from unittest.mock import patch import pytest @@ -50,6 +51,105 @@ async def read_file(self, path: str) -> str | None: return self._files.get(path) +class TestRequestCompressionConfig: + async def test_defaults(self) -> None: + with patch.dict(os.environ, {}, clear=True): + config = await AsyncAwsConfig.resolve( + region="us-west-2", fs=NullFileSystem() + ) + assert config.disable_request_compression is False + assert config.request_min_compression_size_bytes == 10240 + assert config.request_compression_encoding is None + + async def test_shared_profile_settings(self) -> None: + fs = FakeFileSystem( + { + "/fake/config": "[profile compression]\ndisable_request_compression = true\nrequest_min_compression_size_bytes = 256\n" + } + ) + with patch.dict(os.environ, {}, clear=True): + config = await AsyncAwsConfig.resolve( + region="us-west-2", + profile="compression", + fs=fs, + config_file_path="/fake/config", + credentials_file_path="/fake/credentials", + ) + assert config.disable_request_compression is True + assert config.request_min_compression_size_bytes == 256 + + async def test_environment_overrides_profile_and_explicit_overrides_environment( + self, + ) -> None: + fs = FakeFileSystem( + { + "/fake/config": "[default]\ndisable_request_compression = false\nrequest_min_compression_size_bytes = 100\n" + } + ) + with patch.dict( + os.environ, + { + "AWS_DISABLE_REQUEST_COMPRESSION": "true", + "AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "200", + }, + clear=True, + ): + config = await AsyncAwsConfig.resolve( + region="us-west-2", + fs=fs, + config_file_path="/fake/config", + credentials_file_path="/fake/credentials", + ) + assert config.disable_request_compression is True + assert config.request_min_compression_size_bytes == 200 + explicit = await AsyncAwsConfig.resolve( + region="us-west-2", + fs=fs, + config_file_path="/fake/config", + credentials_file_path="/fake/credentials", + disable_request_compression=False, + request_min_compression_size_bytes=0, + request_compression_encoding="gzip", + ) + assert explicit.disable_request_compression is False + assert explicit.request_min_compression_size_bytes == 0 + assert explicit.request_compression_encoding == "gzip" + + @pytest.mark.parametrize("minimum", [-1, 10485761, True, 1.5, "large"]) + async def test_invalid_threshold(self, minimum: Any) -> None: + with patch.dict(os.environ, {}, clear=True): + with pytest.raises( + ConfigValidationError, match="request_min_compression_size_bytes" + ): + await AsyncAwsConfig.resolve( + region="us-west-2", + fs=NullFileSystem(), + request_min_compression_size_bytes=minimum, + ) + + async def test_explicit_gzip_conflicts_with_environment_disable(self) -> None: + with patch.dict( + os.environ, {"AWS_DISABLE_REQUEST_COMPRESSION": "true"}, clear=True + ): + with pytest.raises( + ConfigValidationError, match="disable_request_compression" + ): + await AsyncAwsConfig.resolve( + region="us-west-2", + fs=NullFileSystem(), + request_compression_encoding="gzip", + ) + + async def test_invalid_environment_threshold_is_rejected(self) -> None: + with patch.dict( + os.environ, + {"AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES": "invalid"}, + clear=True, + ): + with pytest.raises(ConfigValidationError, match="Invalid integer"): + await AsyncAwsConfig.resolve(region="us-west-2", fs=NullFileSystem()) + + class TestResolveRegion: @pytest.mark.asyncio async def test_resolves_from_aws_region(self): diff --git a/packages/smithy-core/.changes/next-release/smithy-core-bugfix-4cef458d2ac549dab2645848ab26eb6b.json b/packages/smithy-core/.changes/next-release/smithy-core-bugfix-4cef458d2ac549dab2645848ab26eb6b.json new file mode 100644 index 000000000..930b6b87d --- /dev/null +++ b/packages/smithy-core/.changes/next-release/smithy-core-bugfix-4cef458d2ac549dab2645848ab26eb6b.json @@ -0,0 +1,4 @@ +{ + "type": "bugfix", + "description": "Close asynchronous request-body generators through aclose and await general close awaitables so partial streamed requests release their sources." +} \ No newline at end of file diff --git a/packages/smithy-core/.changes/next-release/smithy-core-feature-76b67ff1b28948f19f2f4ffb34dd75e3.json b/packages/smithy-core/.changes/next-release/smithy-core-feature-76b67ff1b28948f19f2f4ffb34dd75e3.json new file mode 100644 index 000000000..226f099c2 --- /dev/null +++ b/packages/smithy-core/.changes/next-release/smithy-core-feature-76b67ff1b28948f19f2f4ffb34dd75e3.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Add a typed requestCompression trait that preserves the ordered content encodings advertised by an operation." +} \ No newline at end of file diff --git a/packages/smithy-core/src/smithy_core/aio/utils.py b/packages/smithy-core/src/smithy_core/aio/utils.py index 69d2e85c8..4704b3c56 100644 --- a/packages/smithy-core/src/smithy_core/aio/utils.py +++ b/packages/smithy-core/src/smithy_core/aio/utils.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from asyncio import iscoroutine, sleep from collections.abc import AsyncIterable, Iterable +from inspect import isawaitable from typing import Any from ..exceptions import AsyncBodyError @@ -54,9 +55,13 @@ def read_streaming_blob(body: StreamingBlob) -> bytes: async def close(stream: Any) -> None: - """Close a stream, awaiting it if it's async.""" - if (close := getattr(stream, "close", None)) is not None: - if iscoroutine(result := close()): + """Close a stream or async generator, awaiting any returned awaitable.""" + closer = getattr(stream, "close", None) + if closer is None: + closer = getattr(stream, "aclose", None) + if closer is not None: + result = closer() + if isawaitable(result): await result diff --git a/packages/smithy-core/src/smithy_core/traits.py b/packages/smithy-core/src/smithy_core/traits.py index 12c04eb73..4c4f6a7fa 100644 --- a/packages/smithy-core/src/smithy_core/traits.py +++ b/packages/smithy-core/src/smithy_core/traits.py @@ -158,6 +158,19 @@ def __post_init__(self): assert self.document_value is None +@dataclass(init=False, frozen=True) +class RequestCompressionTrait(Trait, id=ShapeID("smithy.api#requestCompression")): + """The priority-ordered content encodings supported by an operation.""" + + @property + def encodings(self) -> list[str]: + assert isinstance(self.document_value, Mapping) + encodings = self.document_value["encodings"] + assert isinstance(encodings, Sequence) and not isinstance(encodings, str) + assert all(isinstance(encoding, str) for encoding in encodings) + return list(encodings) # type: ignore + + @dataclass(init=False, frozen=True) class RequiresLengthTrait(Trait, id=ShapeID("smithy.api#requiresLength")): def __post_init__(self): diff --git a/packages/smithy-core/tests/unit/aio/test_utils.py b/packages/smithy-core/tests/unit/aio/test_utils.py index 2d1138a71..7cd8edf49 100644 --- a/packages/smithy-core/tests/unit/aio/test_utils.py +++ b/packages/smithy-core/tests/unit/aio/test_utils.py @@ -1,7 +1,9 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +from collections.abc import AsyncIterator, Awaitable, Generator from io import BytesIO +import pytest from smithy_core.aio.types import AsyncBytesProvider from smithy_core.aio.utils import close, seek @@ -24,6 +26,42 @@ async def test_close_non_closeable() -> None: await close(b"foo") +async def test_close_async_generator_runs_finalizer() -> None: + closed = False + + async def source() -> AsyncIterator[bytes]: + nonlocal closed + try: + yield b"first" + yield b"second" + finally: + closed = True + + stream = source() + assert await anext(stream) == b"first" + await close(stream) + assert closed + with pytest.raises(StopAsyncIteration): + await anext(stream) + + +async def test_close_awaits_non_coroutine_awaitable() -> None: + closed = False + + class Finalizer: + def __await__(self) -> Generator[None, None, None]: + nonlocal closed + yield + closed = True + + class Stream: + def close(self) -> Awaitable[None]: + return Finalizer() + + await close(Stream()) + assert closed + + async def test_seek_sync_seekable() -> None: stream = BytesIO(b"foo") assert stream.seekable() diff --git a/packages/smithy-core/tests/unit/test_schema_compact.py b/packages/smithy-core/tests/unit/test_schema_compact.py index 3615b9ada..680e7b4d6 100644 --- a/packages/smithy-core/tests/unit/test_schema_compact.py +++ b/packages/smithy-core/tests/unit/test_schema_compact.py @@ -3,12 +3,13 @@ # pyright: reportPrivateUsage=false, reportUnusedFunction=false import threading from collections.abc import Iterator +from types import MappingProxyType import pytest from smithy_core import _schema_compact from smithy_core._schema_compact import StaticSchema, hydrate from smithy_core.shapes import ShapeID, ShapeType -from smithy_core.traits import SensitiveTrait +from smithy_core.traits import RequestCompressionTrait, SensitiveTrait NS = "ns.example" @@ -42,6 +43,21 @@ def test_simple_shape_hydrates_and_caches() -> None: assert hydrate(tup) is schema +def test_request_compression_encodings_read_frozen_compact_trait_value() -> None: + operation: StaticSchema = ( + ShapeType.OPERATION.value, + NS, + "CompressedOperation", + {"smithy.api#requestCompression": MappingProxyType({"encodings": ("gzip",)})}, + _struct("Input", ()), + _struct("Output", ()), + (), + ) + trait = hydrate(operation).expect_trait(RequestCompressionTrait) + assert isinstance(trait.document_value, MappingProxyType) + assert trait.encodings == ["gzip"] + + def test_prelude_schema_passes_through() -> None: from smithy_core import prelude diff --git a/packages/smithy-core/tests/unit/test_traits.py b/packages/smithy-core/tests/unit/test_traits.py index 623387411..cff1b3521 100644 --- a/packages/smithy-core/tests/unit/test_traits.py +++ b/packages/smithy-core/tests/unit/test_traits.py @@ -11,6 +11,7 @@ ErrorFault, ErrorTrait, JSONNameTrait, + RequestCompressionTrait, Trait, UnitTypeTrait, ) @@ -69,3 +70,9 @@ def test_cant_construct_trait_from_non_matching_dynamic_trait(): dynamic = DynamicTrait(id=JSONNameTrait.id, document_value="client") with pytest.raises(ValueError): ErrorTrait(dynamic) + + +def test_request_compression_trait_hydrates_encodings(): + trait = Trait.new(ShapeID("smithy.api#requestCompression"), {"encodings": ["gzip"]}) + assert isinstance(trait, RequestCompressionTrait) + assert trait.encodings == ["gzip"] diff --git a/packages/smithy-http/.changes/next-release/smithy-http-bugfix-2b89be327ad14f599285697d48eb61e3.json b/packages/smithy-http/.changes/next-release/smithy-http-bugfix-2b89be327ad14f599285697d48eb61e3.json new file mode 100644 index 000000000..546826999 --- /dev/null +++ b/packages/smithy-http/.changes/next-release/smithy-http-bugfix-2b89be327ad14f599285697d48eb61e3.json @@ -0,0 +1,4 @@ +{ + "type": "bugfix", + "description": "Probe seekable request bodies with a bounded read and restore their positions instead of scanning to EOF before transmission." +} \ No newline at end of file diff --git a/packages/smithy-http/.changes/next-release/smithy-http-feature-dc6d4536b20e4253a86771e28a53dfd9.json b/packages/smithy-http/.changes/next-release/smithy-http-feature-dc6d4536b20e4253a86771e28a53dfd9.json new file mode 100644 index 000000000..57c16ab7f --- /dev/null +++ b/packages/smithy-http/.changes/next-release/smithy-http-feature-dc6d4536b20e4253a86771e28a53dfd9.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Compress eligible HTTP request bodies with gzip before signing, with configurable thresholds, an explicit encoding choice, and streaming support that preserves seekable sources without buffering the entire compressed payload." +} \ No newline at end of file diff --git a/packages/smithy-http/src/smithy_http/aio/aiohttp.py b/packages/smithy-http/src/smithy_http/aio/aiohttp.py index c122ad8ad..08d529b99 100644 --- a/packages/smithy-http/src/smithy_http/aio/aiohttp.py +++ b/packages/smithy-http/src/smithy_http/aio/aiohttp.py @@ -162,9 +162,11 @@ async def _prepare_body(self, body: StreamingBlob) -> AsyncBytesReader | None: return body position = await body.seek(0, 1) - end = await body.seek(0, 2) - await body.seek(position) - return None if position == end else body + try: + has_data = bool(await body.read(1)) + finally: + await body.seek(position) + return body if has_data else None def _serialize_uri(self, uri: URI) -> yarl.URL: """Serialize the URI, preserving the already-encoded query string as-is.""" diff --git a/packages/smithy-http/src/smithy_http/aio/protocols.py b/packages/smithy-http/src/smithy_http/aio/protocols.py index a1297b332..10790d9be 100644 --- a/packages/smithy-http/src/smithy_http/aio/protocols.py +++ b/packages/smithy-http/src/smithy_http/aio/protocols.py @@ -36,6 +36,7 @@ ) from .. import tuples_to_fields +from ..compression import RequestCompression from ..deserializers import HTTPResponseDeserializer from ..schema_extensions import ( HTTP_BINDING_SCHEMA_EXTENSION, @@ -135,6 +136,9 @@ def serialize_request[ schema=operation.input_schema, http_trait=operation_metadata.http_trait, operation_metadata=operation_metadata, + request_compression=RequestCompression.from_operation( + operation, context + ), ) try: input.serialize_members(serializer) @@ -147,6 +151,7 @@ def serialize_request[ payload_codec=self.payload_codec, http_trait=operation_metadata.http_trait, operation_metadata=operation_metadata, + request_compression=RequestCompression.from_operation(operation, context), ) input.serialize(serializer) if serializer.result is None: @@ -395,6 +400,7 @@ def serialize_request[ endpoint: URI, context: TypedProperties, ) -> HTTPRequest: + compression = RequestCompression.from_operation(operation, context) operation_name = operation.schema.id.name path = f"/service/{self._service_name}/operation/{operation_name}" @@ -412,11 +418,13 @@ def serialize_request[ fields.append(("Content-Type", self._content_type)) fields.append(("Content-Length", str(len(payload)))) + http_fields = tuples_to_fields(fields) + body = compression.compress(payload, http_fields) return _HTTPRequest( method="POST", destination=_URI(host="", path=path), - fields=tuples_to_fields(fields), - body=AsyncBytesReader(payload), + fields=http_fields, + body=AsyncBytesReader(body), ) async def deserialize_response[ diff --git a/packages/smithy-http/src/smithy_http/compression.py b/packages/smithy-http/src/smithy_http/compression.py new file mode 100644 index 000000000..955b373ff --- /dev/null +++ b/packages/smithy-http/src/smithy_http/compression.py @@ -0,0 +1,174 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Modeled HTTP request compression, applied before signing.""" + +import gzip +import zlib +from collections.abc import AsyncGenerator +from dataclasses import dataclass +from typing import Any + +from smithy_core.aio.interfaces import StreamingBlob +from smithy_core.aio.types import AsyncBytesReader +from smithy_core.aio.utils import read_streaming_blob +from smithy_core.interfaces import TypedProperties +from smithy_core.schemas import APIOperation +from smithy_core.shapes import ShapeType +from smithy_core.traits import RequestCompressionTrait, RequiresLengthTrait + +from . import Field +from .interfaces import Fields + + +@dataclass(frozen=True, kw_only=True) +class RequestCompression: + """Compression policy resolved once for a modeled operation.""" + + encoding: str | None = None + minimum_size: int = 10240 + + @classmethod + def from_operation( + cls, operation: APIOperation[Any, Any], context: TypedProperties + ) -> "RequestCompression": + config = context.get("config") + disabled = getattr(config, "disable_request_compression", False) + minimum_size = getattr(config, "request_min_compression_size_bytes", 10240) + explicit = getattr(config, "request_compression_encoding", None) + if not isinstance(disabled, bool): + raise ValueError("disable_request_compression must be a boolean") + if ( + not isinstance(minimum_size, int) + or isinstance(minimum_size, bool) + or not 0 <= minimum_size <= 10485760 + ): + raise ValueError( + "request_min_compression_size_bytes must be an integer between 0 and 10485760" + ) + if explicit not in (None, "gzip"): + raise ValueError("request_compression_encoding must be None or 'gzip'") + if disabled and explicit is not None: + raise ValueError( + "request_compression_encoding conflicts with disable_request_compression" + ) + trait = operation.schema.get_trait(RequestCompressionTrait) + encodings = trait.encodings if trait is not None else [] + supported = any(encoding.lower() == "gzip" for encoding in encodings) + if explicit is not None and not supported: + raise ValueError( + f"Operation {operation.schema.id.name} does not support gzip request compression" + ) + if disabled or not supported: + return cls() + stream = operation.input_stream_member + unsafe_stream = stream is not None and ( + RequiresLengthTrait in stream + or ( + stream.member_target is not None + and stream.member_target.shape_type == ShapeType.UNION + ) + ) + if explicit is not None and unsafe_stream: + raise ValueError( + f"Operation {operation.schema.id.name} does not support gzip request compression" + ) + if unsafe_stream: + return cls() + return cls(encoding="gzip", minimum_size=0 if explicit else minimum_size) + + def compress( + self, body: StreamingBlob, fields: Fields, *, streaming: bool = False + ) -> StreamingBlob: + """Compress buffered content or wrap a stream without reading it eagerly.""" + if self.encoding is None: + return body + if streaming: + result = _GzipReader(body) + if "content-length" in fields: + del fields["content-length"] + else: + payload = read_streaming_blob(body) + if len(payload) < self.minimum_size: + return payload + compressed = gzip.compress(payload, mtime=0) + result = AsyncBytesReader(compressed) + fields.set_field( + Field(name="content-length", values=[str(len(compressed))]) + ) + previous = fields["content-encoding"] if "content-encoding" in fields else None + existing_encoding = previous.as_string() if previous is not None else "" + encoding = f"{existing_encoding}, gzip" if existing_encoding else "gzip" + fields.set_field(Field(name="content-encoding", values=[encoding])) + return result + + +class _GzipReader(AsyncBytesReader): + def __init__(self, body: StreamingBlob) -> None: + self._source = AsyncBytesReader(body) + self._origin = self._source.tell() if self._source.seekable() else None + self._position = 0 + self._compressed = _gzip_chunks(self._source) + super().__init__(self._compressed) + + async def read(self, size: int = -1) -> bytes: + result = await super().read(size) + self._position += len(result) + return result + + def seekable(self) -> bool: + return self._origin is not None + + def tell(self) -> int: + if not self.seekable(): + raise NotImplementedError + if self.closed: + raise ValueError("I/O operation on closed file.") + return self._position + + async def seek(self, offset: int, whence: int = 0) -> int: + """Seek compressed bytes by rewinding and recompressing in bounded chunks. + + Positions beyond EOF are clamped to EOF, as with gzip.GzipFile. + """ + if self._origin is None: + raise NotImplementedError + if self.closed: + raise ValueError("I/O operation on closed file.") + if whence == 0: + target = offset + elif whence == 1: + target = self._position + offset + elif whence == 2: + while await self.read(65536): + pass + target = self._position + offset + else: + raise ValueError(f"Invalid whence: {whence}") + if target < 0: + raise ValueError("Negative seek position") + if target == self._position: + return self._position + await self._compressed.aclose() + await self._source.seek(self._origin) + self._compressed = _gzip_chunks(self._source) + super().__init__(self._compressed) + self._position = 0 + while self._position < target: + if not await self.read(min(65536, target - self._position)): + break + return self._position + + async def close(self) -> None: + try: + await super().close() + finally: + await self._source.close() + + +async def _gzip_chunks(reader: AsyncBytesReader) -> AsyncGenerator[bytes]: + compressor = zlib.compressobj(wbits=31) + async for chunk in reader.iter_chunks(65536): + if encoded := compressor.compress(chunk): + yield encoded + if encoded := compressor.flush(): + yield encoded diff --git a/packages/smithy-http/src/smithy_http/serializers.py b/packages/smithy-http/src/smithy_http/serializers.py index 471cb154b..158ec90ce 100644 --- a/packages/smithy-http/src/smithy_http/serializers.py +++ b/packages/smithy-http/src/smithy_http/serializers.py @@ -39,6 +39,7 @@ from .aio import HTTPResponse as _HTTPResponse from .aio.interfaces import HTTPRequest, HTTPResponse from .bindings import Binding, RequestBindingMatcher +from .compression import RequestCompression from .schema_extensions import ( HTTP_BINDING_SCHEMA_EXTENSION, HTTPBindingSchemaMetadata, @@ -73,6 +74,7 @@ def __init__( endpoint_trait: EndpointTrait | None = None, omit_empty_payload: bool = True, operation_metadata: HTTPOperationSchemaMetadata | None = None, + request_compression: RequestCompression | None = None, ) -> None: """Initialize an HTTPRequestSerializer. @@ -89,6 +91,7 @@ def __init__( self._payload_codec = payload_codec self._omit_empty_payload = omit_empty_payload self._operation_metadata = operation_metadata + self._request_compression = request_compression self.result: HTTPRequest | None = None @contextmanager @@ -100,6 +103,7 @@ def begin_struct(self, schema: Schema) -> Iterator[ShapeSerializer]: endpoint_trait=self._endpoint_trait, omit_empty_payload=self._omit_empty_payload, operation_metadata=self._operation_metadata, + request_compression=self._request_compression, ) try: yield binding_serializer @@ -149,6 +153,7 @@ def __init__( endpoint_trait: EndpointTrait | None = None, omit_empty_payload: bool = True, operation_metadata: HTTPOperationSchemaMetadata | None = None, + request_compression: RequestCompression | None = None, ) -> None: """Initialize an HTTPBindingSerializer. @@ -159,6 +164,7 @@ def __init__( :param omit_empty_payload: Whether an empty document payload should be omitted. :param operation_metadata: Optional cached operation binding metadata. """ + self._request_compression = request_compression self._operation_metadata = operation_metadata or ( HTTPOperationSchemaMetadata.from_traits(http_trait, endpoint_trait) ) @@ -289,6 +295,24 @@ def build_request(self) -> HTTPRequest: ) fields.set_field(Field(name="content-length", values=[str(content_length)])) + if self._request_compression is not None: + raw_body = ( + self._raw_payload_serializer.payload or b"" + if self._raw_payload_serializer is not None + else self._sync_payload + ) + if raw_body is not None: + payload = AsyncBytesReader( + self._request_compression.compress( + raw_body, + fields, + streaming=( + self._binding_metadata.request.payload is not None + and self._binding_metadata.request.payload.is_streaming + ), + ) + ) + self._result = _HTTPRequest( method=self._operation_metadata.method, destination=URI( diff --git a/packages/smithy-http/tests/unit/aio/test_aiohttp.py b/packages/smithy-http/tests/unit/aio/test_aiohttp.py index 4a2d424ee..55ceb0098 100644 --- a/packages/smithy-http/tests/unit/aio/test_aiohttp.py +++ b/packages/smithy-http/tests/unit/aio/test_aiohttp.py @@ -1,7 +1,11 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 # pyright: reportPrivateUsage=false +import gzip +from asyncio import CancelledError from collections.abc import AsyncIterator +from hashlib import sha256 +from io import BytesIO from typing import Any, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -11,6 +15,7 @@ from smithy_http import Field, Fields from smithy_http.aio import HTTPRequest from smithy_http.aio.aiohttp import AIOHTTPClient, _AIOHTTPStreamingBody +from smithy_http.compression import RequestCompression from smithy_http.exceptions import SmithyHTTPError @@ -86,6 +91,83 @@ async def test_send_preserves_explicitly_framed_empty_body() -> None: assert session.request.call_args.kwargs["data"] is body +@pytest.mark.parametrize("position", [0, 17]) +async def test_send_prepares_compressed_seekable_body_with_bounded_reads( + position: int, +) -> None: + class CountingSource(BytesIO): + bytes_read = 0 + + def read(self, size: int | None = -1) -> bytes: + data = super().read(size) + self.bytes_read += len(data) + return data + + payload = b"".join( + sha256(index.to_bytes(4, "big")).digest() for index in range(32768) + ) + source = CountingSource(payload) + fields = Fields() + body = RequestCompression(encoding="gzip", minimum_size=0).compress( + source, fields, streaming=True + ) + assert isinstance(body, AsyncBytesReader) + await body.seek(position) + source.bytes_read = 0 + client, session = _create_client() + request = HTTPRequest( + method="POST", + destination=URI(scheme="https", host="example.com"), + body=body, + fields=fields, + ) + + await client.send(request) + + assert source.bytes_read <= 128 * 1024 + assert session.request.call_args.kwargs["data"] is body + assert body.tell() == position + await body.seek(0) + assert gzip.decompress(await body.read()) == payload + await body.close() + await client.close() + + +async def test_send_omits_body_at_nonzero_end_position() -> None: + client, session = _create_client() + body = AsyncBytesReader(b"prefix") + await body.seek(6) + request = _create_request() + request.body = body + await client.send(request) + assert session.request.call_args.kwargs["data"] is None + assert body.tell() == 6 + await body.close() + await client.close() + + +@pytest.mark.parametrize("error", [OSError, CancelledError]) +async def test_send_restores_position_when_body_probe_fails( + error: type[BaseException], +) -> None: + class FailingProbeReader(AsyncBytesReader): + async def read(self, size: int = -1) -> bytes: + await super().read(size) + raise error("probe failed") + + client, session = _create_client() + body = FailingProbeReader(b"payload") + await body.seek(2) + request = _create_request() + request.body = body + with pytest.raises(error, match="probe failed"): + await client.send(request) + assert body.tell() == 2 + session.request.assert_not_awaited() + await body.close() + await client.close() + + async def test_send_disables_redirects() -> None: client, session = _create_client() diff --git a/packages/smithy-http/tests/unit/aio/test_protocols.py b/packages/smithy-http/tests/unit/aio/test_protocols.py index 775bd22ff..e3008d67f 100644 --- a/packages/smithy-http/tests/unit/aio/test_protocols.py +++ b/packages/smithy-http/tests/unit/aio/test_protocols.py @@ -1,10 +1,13 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 +import gzip +from types import SimpleNamespace from typing import Any, Self import pytest from smithy_core import URI +from smithy_core.aio.types import AsyncBytesReader from smithy_core.codecs import Codec from smithy_core.deserializers import ShapeDeserializer from smithy_core.documents import TypeRegistry @@ -149,6 +152,101 @@ async def read(self, size: int = -1) -> bytes: return b"{}" +async def test_http_binding_compresses_absent_optional_payload_as_empty() -> None: + class OptionalPayloadInput: + SCHEMA = Schema.collection( + id=ShapeID("ns.foo#OptionalPayloadInput"), + members={ + "payload": { + "target": Schema( + id=ShapeID("ns.foo#Payload"), shape_type=ShapeType.BLOB + ), + "traits": [HTTPPayloadTrait()], + } + }, + ) + + def serialize(self, serializer: ShapeSerializer) -> None: + serializer.write_struct(self.SCHEMA, self) + + def serialize_members(self, serializer: ShapeSerializer) -> None: + pass + + operation = APIOperation( + input=OptionalPayloadInput, + output=MockOutput, + static_schema=( + ShapeType.OPERATION.value, + "ns.foo", + "OptionalPayloadOperation", + { + "smithy.api#http": {"method": "POST", "code": 200, "uri": "/"}, + "smithy.api#requestCompression": {"encodings": ["gzip"]}, + }, + OptionalPayloadInput.SCHEMA, + MockOutput.SCHEMA, + (), + ), + error_registry=TypeRegistry({}), + effective_auth_schemes=[], + ) + request = MockBindingProtocol().serialize_request( + operation=operation, + input=OptionalPayloadInput(), + endpoint=URI(host="example.com"), + context=TypedProperties( + {"config": SimpleNamespace(request_compression_encoding="gzip")} + ), + ) + assert request.fields["Content-Encoding"].as_string() == "gzip" + wire = await AsyncBytesReader(request.body).read() + assert gzip.decompress(wire) == b"" + assert int(request.fields["Content-Length"].as_string()) == len(wire) + + +@pytest.mark.parametrize("explicit", [None, "gzip"]) +async def test_http_binding_protocol_compresses_modeled_request( + explicit: str | None, +) -> None: + operation = APIOperation( + input=StructInput, + output=MockOutput, + static_schema=( + ShapeType.OPERATION.value, + "ns.foo", + "CompressedOperation", + { + "smithy.api#http": { + "method": "POST", + "code": 200, + "uri": "/compressed", + }, + "smithy.api#requestCompression": {"encodings": ["gzip"]}, + }, + StructInput.SCHEMA, + MockOutput.SCHEMA, + (), + ), + error_registry=TypeRegistry({}), + effective_auth_schemes=[], + ) + config = SimpleNamespace( + request_compression_encoding=explicit, + disable_request_compression=False, + request_min_compression_size_bytes=0 if explicit is None else 10240, + ) + request = MockBindingProtocol().serialize_request( + operation=operation, + input=StructInput(), + endpoint=URI(host="example.com"), + context=TypedProperties({"config": config}), + ) + assert request.fields["Content-Encoding"].as_string() == "gzip" + wire_body = await AsyncBytesReader(request.body).read() + assert gzip.decompress(wire_body) == b"" + assert int(request.fields["Content-Length"].as_string()) == len(wire_body) + + @pytest.mark.parametrize("body", [b"{}", bytearray(b"{}")]) async def test_buffer_async_body_fast_paths_synchronous_bytes( monkeypatch: pytest.MonkeyPatch, diff --git a/packages/smithy-http/tests/unit/test_compression.py b/packages/smithy-http/tests/unit/test_compression.py new file mode 100644 index 000000000..25056ed4e --- /dev/null +++ b/packages/smithy-http/tests/unit/test_compression.py @@ -0,0 +1,195 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +import gzip +from collections.abc import AsyncIterator +from io import BytesIO +from types import SimpleNamespace +from typing import Any, cast + +import pytest +from smithy_core.aio.types import AsyncBytesReader +from smithy_core.aio.utils import close +from smithy_core.schemas import APIOperation, Schema +from smithy_core.shapes import ShapeID, ShapeType +from smithy_core.traits import ( + DynamicTrait, + RequestCompressionTrait, + RequiresLengthTrait, + Trait, +) +from smithy_core.types import TypedProperties +from smithy_http import Field, Fields +from smithy_http.compression import RequestCompression + + +def make_policy( + *, + encodings: list[str] | None = None, + modeled: bool = True, + explicit: str | None = None, + disabled: bool = False, + threshold: int = 10240, + stream: Schema | None = None, +) -> RequestCompression: + traits: list[Trait | DynamicTrait] = ( + [RequestCompressionTrait({"encodings": encodings or ["gzip"]})] + if modeled + else [] + ) + operation = SimpleNamespace( + schema=Schema( + id=ShapeID("example#Compress"), + shape_type=ShapeType.OPERATION, + traits=traits, + ), + input_schema=Schema.collection(id=ShapeID("example#Input")), + input_stream_member=stream, + ) + context = TypedProperties( + { + "config": SimpleNamespace( + request_compression_encoding=explicit, + disable_request_compression=disabled, + request_min_compression_size_bytes=threshold, + ) + } + ) + return RequestCompression.from_operation( + cast(APIOperation[Any, Any], operation), context + ) + + +@pytest.mark.parametrize( + "payload", [b"", b"hello", bytearray(b"hello"), BytesIO(b"hello")] +) +async def test_explicit_gzip_round_trip_below_threshold( + payload: bytes | bytearray | BytesIO, +) -> None: + expected = payload.getvalue() if isinstance(payload, BytesIO) else payload + fields = Fields([Field(name="Content-Length", values=["999"])]) + result = make_policy(explicit="gzip").compress(payload, fields) + wire = await AsyncBytesReader(result).read() + assert gzip.decompress(wire) == expected + assert fields["Content-Encoding"].as_string() == "gzip" + assert int(fields["Content-Length"].as_string()) == len(wire) + + +@pytest.mark.parametrize( + "size, compressed", [(10239, False), (10240, True), (10241, True)] +) +async def test_automatic_compression_threshold(size: int, compressed: bool) -> None: + payload = b"x" * size + fields = Fields() + result = make_policy().compress(payload, fields) + wire = await AsyncBytesReader(result).read() + if compressed: + assert gzip.decompress(wire) == payload + assert fields["Content-Encoding"].as_string() == "gzip" + else: + assert wire == payload + assert "Content-Encoding" not in fields + + +@pytest.mark.parametrize( + "kwargs", [{"modeled": False}, {"disabled": True}, {"encodings": ["unknown"]}] +) +async def test_automatic_compression_skips_ineligible_request( + kwargs: dict[str, Any], +) -> None: + payload = b"x" * 11000 + fields = Fields() + result = make_policy(**kwargs).compress(payload, fields) + assert await AsyncBytesReader(result).read() == payload + assert "Content-Encoding" not in fields + + +@pytest.mark.parametrize( + "kwargs", [{"modeled": False}, {"disabled": True}, {"encodings": ["unknown"]}] +) +def test_explicit_gzip_fails_instead_of_silent_fallback(kwargs: dict[str, Any]) -> None: + with pytest.raises(ValueError, match=r"gzip|compression"): + make_policy(explicit="gzip", **kwargs).compress(b"hello", Fields()) + + +def test_unknown_explicit_codec_is_rejected() -> None: + with pytest.raises(ValueError, match=r"zstd|encoding"): + make_policy(explicit="zstd").compress(b"hello", Fields()) + + +async def test_gzip_appends_existing_content_encoding() -> None: + fields = Fields([Field(name="Content-Encoding", values=["identity"])]) + wire = await AsyncBytesReader( + make_policy(explicit="gzip").compress(b"hello", fields) + ).read() + assert gzip.decompress(wire) == b"hello" + assert "identity" in fields["Content-Encoding"].as_string() + assert "gzip" in fields["Content-Encoding"].as_string() + + +async def test_streaming_gzip_is_lazy_and_lossless() -> None: + reads: list[bytes] = [] + + async def source() -> AsyncIterator[bytes]: + for part in (b"first", b"", b"second" * 20000): + reads.append(part) + yield part + + fields = Fields([Field(name="Content-Length", values=["123"])]) + result = make_policy().compress(source(), fields, streaming=True) + assert reads == [] + assert "Content-Length" not in fields + wire = await AsyncBytesReader(result).read() + assert gzip.decompress(wire) == b"first" + b"second" * 20000 + assert len(reads) == 3 + + +async def test_closing_compressed_stream_closes_underlying_source() -> None: + source = BytesIO(b"hello") + result = make_policy().compress(source, Fields(), streaming=True) + await close(result) + assert source.closed + + +async def test_closing_partially_read_compressed_async_generator_closes_source() -> ( + None +): + closed = False + + async def source() -> AsyncIterator[bytes]: + nonlocal closed + try: + yield b"x" * 65536 + yield b"second chunk" + finally: + closed = True + + stream = source() + result = AsyncBytesReader(make_policy().compress(stream, Fields(), streaming=True)) + assert await result.read(1) + assert not closed + await close(result) + assert closed + with pytest.raises(StopAsyncIteration): + await anext(stream) + + +@pytest.mark.parametrize("kind", ["requires_length", "eventstream"]) +async def test_incompatible_streams_are_skipped_or_rejected(kind: str) -> None: + target = Schema( + id=ShapeID("example#Stream"), + shape_type=ShapeType.BLOB if kind == "requires_length" else ShapeType.UNION, + traits=[RequiresLengthTrait()] if kind == "requires_length" else [], + ) + stream = Schema.member(ShapeID("example#Input$payload"), target, 0) + fields = Fields() + body = b"wire-stream" + assert ( + await AsyncBytesReader( + make_policy(stream=stream).compress(body, fields, streaming=True) + ).read() + == body + ) + assert "Content-Encoding" not in fields + with pytest.raises(ValueError, match="gzip"): + make_policy(explicit="gzip", stream=stream) From a0c29187a18aa9407947b81290c4c63fc882b27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Silva=20Gouv=C3=AAa?= <64869705+gouveags@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:35:46 -0300 Subject: [PATCH 2/5] Preserve async stream positions across compression errors and signing --- .../tests/unit/aio/test_protocols.py | 30 ++++++++++++-- .../src/smithy_http/compression.py | 18 +++++++-- .../tests/unit/aio/test_aiohttp.py | 39 +++++++++++++++++++ 3 files changed, 80 insertions(+), 7 deletions(-) diff --git a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py index 84928d0ea..f5982a4e8 100644 --- a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py +++ b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py @@ -384,12 +384,36 @@ async def send(self, request: Any, **kwargs: Any) -> HTTPResponse: assert canonical_payloads == [sha256(transmitted[0]).hexdigest()] * 2 -async def test_signed_compressed_seekable_stream_is_not_fully_buffered() -> None: +@pytest.mark.parametrize("source_kind", ["sync", "async_no_tell"]) +async def test_signed_compressed_seekable_stream_is_not_fully_buffered( + source_kind: str, +) -> None: + class AsyncSeekableNoTell: + def __init__(self, data: bytes) -> None: + self.buffer = BytesIO(data) + + async def read(self, size: int = -1) -> bytes: + return self.buffer.read(size) + + async def seek(self, offset: int, whence: int = 0) -> int: + return self.buffer.seek(offset, whence) + + async def close(self) -> None: + self.buffer.close() + + @property + def closed(self) -> bool: + return self.buffer.closed + payload = b"".join( sha256(index.to_bytes(4, "big")).digest() for index in range(8192) ) - source = BytesIO(b"prefix" + payload) - source.seek(6) + if source_kind == "sync": + source = BytesIO(b"prefix" + payload) + source.seek(6) + else: + source = AsyncSeekableNoTell(b"prefix" + payload) + await source.seek(6) fields = Fields() body = RequestCompression(encoding="gzip", minimum_size=0).compress( source, fields, streaming=True diff --git a/packages/smithy-http/src/smithy_http/compression.py b/packages/smithy-http/src/smithy_http/compression.py index 955b373ff..97c787d37 100644 --- a/packages/smithy-http/src/smithy_http/compression.py +++ b/packages/smithy-http/src/smithy_http/compression.py @@ -105,18 +105,23 @@ def compress( class _GzipReader(AsyncBytesReader): def __init__(self, body: StreamingBlob) -> None: self._source = AsyncBytesReader(body) - self._origin = self._source.tell() if self._source.seekable() else None + self._origin: int | None = None self._position = 0 self._compressed = _gzip_chunks(self._source) super().__init__(self._compressed) + async def _ensure_origin(self) -> None: + if self._origin is None and self._source.seekable(): + self._origin = await self._source.seek(0, 1) + async def read(self, size: int = -1) -> bytes: + await self._ensure_origin() result = await super().read(size) self._position += len(result) return result def seekable(self) -> bool: - return self._origin is not None + return self._source.seekable() def tell(self) -> int: if not self.seekable(): @@ -130,10 +135,13 @@ async def seek(self, offset: int, whence: int = 0) -> int: Positions beyond EOF are clamped to EOF, as with gzip.GzipFile. """ - if self._origin is None: + if not self.seekable(): raise NotImplementedError if self.closed: raise ValueError("I/O operation on closed file.") + await self._ensure_origin() + if self._origin is None: + raise NotImplementedError("The source did not return its seek position") if whence == 0: target = offset elif whence == 1: @@ -146,7 +154,9 @@ async def seek(self, offset: int, whence: int = 0) -> int: raise ValueError(f"Invalid whence: {whence}") if target < 0: raise ValueError("Negative seek position") - if target == self._position: + # A failed read may advance the source without emitting compressed bytes. + # Absolute seeks must restart it even when the logical position matches. + if target == self._position and whence != 0: return self._position await self._compressed.aclose() await self._source.seek(self._origin) diff --git a/packages/smithy-http/tests/unit/aio/test_aiohttp.py b/packages/smithy-http/tests/unit/aio/test_aiohttp.py index 55ceb0098..bc346b758 100644 --- a/packages/smithy-http/tests/unit/aio/test_aiohttp.py +++ b/packages/smithy-http/tests/unit/aio/test_aiohttp.py @@ -168,6 +168,45 @@ async def read(self, size: int = -1) -> bytes: await client.close() +@pytest.mark.parametrize("error", [OSError, CancelledError]) +async def test_send_rewinds_gzip_source_after_probe_consumes_then_fails( + error: type[BaseException], +) -> None: + class FailOnceSource(BytesIO): + failed = False + + def read(self, size: int | None = -1) -> bytes: + if not self.failed: + self.failed = True + super().read(1) + raise error("source probe failed") + return super().read(size) + + payload = b"complete gzip source after retry" + source = FailOnceSource(b"prefix" + payload) + source.seek(6) + fields = Fields() + body = RequestCompression(encoding="gzip", minimum_size=0).compress( + source, fields, streaming=True + ) + assert isinstance(body, AsyncBytesReader) + client, session = _create_client() + request = HTTPRequest( + method="POST", + destination=URI(scheme="https", host="example.com"), + body=body, + fields=fields, + ) + with pytest.raises(error, match="source probe failed"): + await client.send(request) + session.request.assert_not_awaited() + assert source.tell() == 6 + assert body.tell() == 0 + assert gzip.decompress(await body.read()) == payload + await body.close() + await client.close() + + async def test_send_disables_redirects() -> None: client, session = _create_client() From ae0e7da0fdac9d8fc49884e539b7cf71eacb041b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Silva=20Gouv=C3=AAa?= <64869705+gouveags@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:42:43 -0300 Subject: [PATCH 3/5] Preserve buffered async iterator data when seeking request bodies --- ...gfix-fc34d7874952413c9fadde9b46832f5b.json | 4 ++ .../smithy-core/src/smithy_core/aio/types.py | 10 +++- .../smithy-core/tests/unit/aio/test_types.py | 56 ++++++++++++++++++ .../tests/unit/aio/test_aiohttp.py | 58 ++++++++++++++++++- 4 files changed, 125 insertions(+), 3 deletions(-) create mode 100644 packages/smithy-core/.changes/next-release/smithy-core-bugfix-fc34d7874952413c9fadde9b46832f5b.json diff --git a/packages/smithy-core/.changes/next-release/smithy-core-bugfix-fc34d7874952413c9fadde9b46832f5b.json b/packages/smithy-core/.changes/next-release/smithy-core-bugfix-fc34d7874952413c9fadde9b46832f5b.json new file mode 100644 index 000000000..e5df996bf --- /dev/null +++ b/packages/smithy-core/.changes/next-release/smithy-core-bugfix-fc34d7874952413c9fadde9b46832f5b.json @@ -0,0 +1,4 @@ +{ + "type": "bugfix", + "description": "Preserve the logical cursor and unread bytes when seeking asynchronous iterable request bodies." +} \ No newline at end of file diff --git a/packages/smithy-core/src/smithy_core/aio/types.py b/packages/smithy-core/src/smithy_core/aio/types.py index 3b84317cd..6b1354e85 100644 --- a/packages/smithy-core/src/smithy_core/aio/types.py +++ b/packages/smithy-core/src/smithy_core/aio/types.py @@ -89,13 +89,19 @@ async def seek(self, offset: int, whence: int = 0) -> int: if not self._seekable: raise NotImplementedError - return await seek(self._data, offset, whence) # type: ignore + # Iterables can yield more bytes than read() returned to the caller. + # Relative seeks start at the logical cursor, before that unread tail. + if whence == 1: + offset -= len(self._remainder) + position = await seek(self._data, offset, whence) + self._remainder = b"" + return position # type: ignore def tell(self) -> int: """Returns the position of the cursor.""" if self._seekable: if (tell := getattr(self._data, "tell", None)) is not None: - return tell() + return tell() - len(self._remainder) raise NotImplementedError async def _read_from_iterable( diff --git a/packages/smithy-core/tests/unit/aio/test_types.py b/packages/smithy-core/tests/unit/aio/test_types.py index d141bbf6d..085465d9c 100644 --- a/packages/smithy-core/tests/unit/aio/test_types.py +++ b/packages/smithy-core/tests/unit/aio/test_types.py @@ -29,6 +29,62 @@ async def __anext__(self) -> bytes: raise StopAsyncIteration +class _SeekableAsyncIteratorWrapper(_AsyncIteratorWrapper): + async def seek(self, offset: int, whence: int = 0) -> int: + return self._source.seek(offset, whence) + + def tell(self) -> int: + return self._source.tell() + + +async def test_seekable_async_iterator_rewind_discards_buffered_remainder() -> None: + reader = AsyncBytesReader(_SeekableAsyncIteratorWrapper(BytesIO(b"payload"))) + assert await reader.read(1) == b"p" + assert await reader.seek(0) == 0 + assert await reader.read() == b"payload" + + +async def test_seekable_async_iterator_reports_logical_position() -> None: + reader = AsyncBytesReader(_SeekableAsyncIteratorWrapper(BytesIO(b"payload"))) + assert await reader.read(1) == b"p" + assert reader.tell() == 1 + assert await reader.seek(0, 1) == 1 + assert await reader.read() == b"ayload" + + +@pytest.mark.parametrize( + "offset, whence, expected", [(1, 1, 2), (-1, 1, 0), (-2, 2, 5)] +) +async def test_seekable_async_iterator_seek_uses_logical_position( + offset: int, whence: int, expected: int +) -> None: + payload = b"payload" + reader = AsyncBytesReader(_SeekableAsyncIteratorWrapper(BytesIO(payload))) + assert await reader.read(1) == b"p" + assert await reader.seek(offset, whence) == expected + assert reader.tell() == expected + assert await reader.read() == payload[expected:] + + +async def test_seekable_async_iterator_failed_seek_preserves_buffer() -> None: + reader = AsyncBytesReader(_SeekableAsyncIteratorWrapper(BytesIO(b"payload"))) + assert await reader.read(1) == b"p" + with pytest.raises(ValueError): + await reader.seek(-1) + assert reader.tell() == 1 + assert await reader.read() == b"ayload" + + +async def test_seekable_async_iterator_rewinds_nonzero_source_origin() -> None: + source = BytesIO(b"prefixpayload") + source.seek(6) + reader = AsyncBytesReader(_SeekableAsyncIteratorWrapper(source)) + assert await reader.read(1) == b"p" + assert reader.tell() == 7 + assert await reader.seek(6) == 6 + assert await reader.read() == b"payload" + + async def test_read_bytes() -> None: reader = AsyncBytesReader(b"foo") assert reader.seekable() diff --git a/packages/smithy-http/tests/unit/aio/test_aiohttp.py b/packages/smithy-http/tests/unit/aio/test_aiohttp.py index bc346b758..84e2773ea 100644 --- a/packages/smithy-http/tests/unit/aio/test_aiohttp.py +++ b/packages/smithy-http/tests/unit/aio/test_aiohttp.py @@ -6,7 +6,7 @@ from collections.abc import AsyncIterator from hashlib import sha256 from io import BytesIO -from typing import Any, cast +from typing import Any, Self, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -207,6 +207,62 @@ def read(self, size: int | None = -1) -> bytes: await client.close() +@pytest.mark.parametrize("compressed", [False, True]) +async def test_send_preserves_seekable_async_iterable_payload(compressed: bool) -> None: + class SeekableChunks: + def __init__(self, payload: bytes) -> None: + self.source = BytesIO(payload) + + def __aiter__(self) -> Self: + return self + + async def __anext__(self) -> bytes: + chunk = self.source.read() + if not chunk: + raise StopAsyncIteration + return chunk + + async def seek(self, offset: int, whence: int = 0) -> int: + return self.source.seek(offset, whence) + + def tell(self) -> int: + return self.source.tell() + + async def close(self) -> None: + self.source.close() + + payload = b"".join( + sha256(index.to_bytes(4, "big")).digest() for index in range(8192) + ) + source = SeekableChunks(payload) + fields = Fields() + body = ( + RequestCompression(encoding="gzip", minimum_size=0).compress( + source, fields, streaming=True + ) + if compressed + else AsyncBytesReader(source) + ) + assert isinstance(body, AsyncBytesReader) + client, session = _create_client() + request = HTTPRequest( + method="POST", + destination=URI(scheme="https", host="example.com"), + body=body, + fields=fields, + ) + await client.send(request) + assert session.request.call_args.kwargs["data"] is body + wire = await body.read() + actual = gzip.decompress(wire) if compressed else wire + assert actual == payload + await body.seek(0) + replay = await body.read() + assert replay == wire + await body.close() + await client.close() + + async def test_send_disables_redirects() -> None: client, session = _create_client() From 800d40ecc45e2f134a38a9625ba24dd46725d3c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Silva=20Gouv=C3=AAa?= <64869705+gouveags@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:27:27 -0300 Subject: [PATCH 4/5] Align request compression with standard SDK configuration --- .../rpcv2-cbor/src/rpcv2cbor/config.py | 5 ---- .../RequestCompressionIntegration.java | 5 ---- packages/smithy-aws-core/README.md | 6 +--- .../src/smithy_aws_core/config/aws_config.py | 17 ----------- .../src/smithy_aws_core/config/validators.py | 8 ----- .../tests/unit/aio/test_protocols.py | 13 ++------- .../tests/unit/config/test_resolver.py | 17 ++--------- .../src/smithy_http/compression.py | 17 +---------- .../tests/unit/aio/test_protocols.py | 10 ++----- .../tests/unit/test_compression.py | 29 +++++-------------- 10 files changed, 17 insertions(+), 110 deletions(-) diff --git a/codegen-output/rpcv2-cbor/src/rpcv2cbor/config.py b/codegen-output/rpcv2-cbor/src/rpcv2cbor/config.py index 1aae7272f..711f80287 100644 --- a/codegen-output/rpcv2-cbor/src/rpcv2cbor/config.py +++ b/codegen-output/rpcv2-cbor/src/rpcv2cbor/config.py @@ -125,9 +125,6 @@ class Config: the service endpoint. """ - request_compression_encoding: str | None - """Explicitly request gzip regardless of size; unsupported operations fail.""" - request_min_compression_size_bytes: int """ Minimum buffered request size for automatic compression; zero compresses @@ -171,7 +168,6 @@ def __init__( | ProtocolConstructor[ClientProtocol[Any, Any]] | None = None, region: str | None = None, - request_compression_encoding: str | None = None, request_min_compression_size_bytes: int | None = None, retry_strategy: RetryStrategy | RetryStrategyOptions | None = None, sdk_ua_app_id: str | None = None, @@ -197,7 +193,6 @@ def __init__( "ClientProtocol[Any, Any]", protocol ) or RpcV2CborClientProtocol(_PROTOCOL_SETTINGS) self.region = region - self.request_compression_encoding = request_compression_encoding self.request_min_compression_size_bytes = ( request_min_compression_size_bytes if request_min_compression_size_bytes is not None diff --git a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RequestCompressionIntegration.java b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RequestCompressionIntegration.java index 5d81ea54a..39f0664cf 100644 --- a/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RequestCompressionIntegration.java +++ b/codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RequestCompressionIntegration.java @@ -39,11 +39,6 @@ public List getClientPlugins(GenerationContext context) { + "request_min_compression_size_bytes if request_min_compression_size_bytes is not None " + "else 10240")) .build()) - .addConfigProperty(ConfigProperty.builder() - .name("request_compression_encoding") - .type(Symbol.builder().name("str").build()) - .documentation("Explicitly request gzip regardless of size; unsupported operations fail.") - .build()) .build()); } } diff --git a/packages/smithy-aws-core/README.md b/packages/smithy-aws-core/README.md index 07294e30a..ab9f79c7a 100644 --- a/packages/smithy-aws-core/README.md +++ b/packages/smithy-aws-core/README.md @@ -23,12 +23,8 @@ AWS client configuration accepts these settings: - `disable_request_compression=True` disables automatic compression. - `request_min_compression_size_bytes=0` compresses every eligible buffered request. -- `request_compression_encoding="gzip"` explicitly requests gzip regardless of size. - An unsupported operation raises an error, as does combining this option with - `disable_request_compression=True`. The standard settings also resolve from `AWS_DISABLE_REQUEST_COMPRESSION` and `AWS_REQUEST_MIN_COMPRESSION_SIZE_BYTES`, or the corresponding lowercase keys in the shared AWS configuration file. Explicit client settings take precedence. -The explicit encoding option is a client setting only. Zstd is not currently an -encoding supported by Smithy's request-compression trait. +Operations without gzip compression metadata remain uncompressed. diff --git a/packages/smithy-aws-core/src/smithy_aws_core/config/aws_config.py b/packages/smithy-aws-core/src/smithy_aws_core/config/aws_config.py index 88b0e76d2..1ea9e1c3f 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/config/aws_config.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/config/aws_config.py @@ -38,7 +38,6 @@ validate_max_attempts, validate_profile, validate_region, - validate_request_compression_encoding, validate_request_min_compression_size_bytes, validate_retry_mode, ) @@ -54,7 +53,6 @@ class AwsConfigOverrides(TypedDict, total=False): max_attempts: int | None disable_request_compression: bool request_min_compression_size_bytes: int - request_compression_encoding: str | None endpoint_uri: "str | URI | None" aws_access_key_id: str | None aws_secret_access_key: str | None @@ -102,9 +100,6 @@ class AsyncAwsConfig: request_min_compression_size_bytes: int = 10240 """Minimum buffered request size for automatic compression; zero compresses all.""" - request_compression_encoding: str | None = None - """Explicitly request gzip regardless of size; unsupported operations fail.""" - endpoint_uri: "str | URI | None" = None """A static URI to route requests to.""" @@ -190,10 +185,6 @@ class AsyncAwsConfig: resolver=resolve_request_min_compression_size_bytes, validator=validate_request_min_compression_size_bytes, ), - "request_compression_encoding": FieldSpec( - default=None, - validator=validate_request_compression_encoding, - ), "endpoint_uri": FieldSpec( default=None, resolver=resolve_endpoint_uri, @@ -383,14 +374,6 @@ async def _resolve_fields(self, overrides: dict[str, Any]) -> None: if spec.validator is not None: spec.validator(getattr(self, field_name)) - if ( - self.disable_request_compression - and self.request_compression_encoding is not None - ): - raise ConfigValidationError( - "request_compression_encoding conflicts with disable_request_compression" - ) - def _apply_default(self, field_name: str, spec: FieldSpec) -> None: """Apply the default value for a field.""" if spec.default_factory is not None: diff --git a/packages/smithy-aws-core/src/smithy_aws_core/config/validators.py b/packages/smithy-aws-core/src/smithy_aws_core/config/validators.py index 326f28072..4360bb3a9 100644 --- a/packages/smithy-aws-core/src/smithy_aws_core/config/validators.py +++ b/packages/smithy-aws-core/src/smithy_aws_core/config/validators.py @@ -121,11 +121,3 @@ def validate_request_min_compression_size_bytes(value: object) -> None: raise ConfigValidationError( "request_min_compression_size_bytes must be an integer between 0 and 10485760" ) - - -def validate_request_compression_encoding(value: object) -> None: - """Only the gzip codec is currently modeled by Smithy.""" - if value not in (None, "gzip"): - raise ConfigValidationError( - "request_compression_encoding must be None or 'gzip'" - ) diff --git a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py index f5982a4e8..b92873fa9 100644 --- a/packages/smithy-aws-core/tests/unit/aio/test_protocols.py +++ b/packages/smithy-aws-core/tests/unit/aio/test_protocols.py @@ -246,10 +246,7 @@ async def test_aws_json11_serializes_base_request_shape() -> None: @pytest.mark.parametrize("kind", ["json", "query"]) -@pytest.mark.parametrize("explicit", [None, "gzip"]) -async def test_aws_protocol_compresses_serialized_request( - kind: str, explicit: str | None -) -> None: +async def test_aws_protocol_compresses_serialized_request(kind: str) -> None: protocol = ( _aws_json11_protocol() if kind == "json" @@ -275,10 +272,7 @@ async def test_aws_protocol_compresses_serialized_request( { "config": SimpleNamespace( disable_request_compression=False, - request_min_compression_size_bytes=0 - if explicit is None - else 10485760, - request_compression_encoding=explicit, + request_min_compression_size_bytes=0, ) } ), @@ -343,8 +337,7 @@ async def send(self, request: Any, **kwargs: Any) -> HTTPResponse: ) ), disable_request_compression=False, - request_min_compression_size_bytes=10240, - request_compression_encoding="gzip", + request_min_compression_size_bytes=0, ) operation = APIOperation( input=_TestInput, diff --git a/packages/smithy-aws-core/tests/unit/config/test_resolver.py b/packages/smithy-aws-core/tests/unit/config/test_resolver.py index 8594c4197..a5078e98d 100644 --- a/packages/smithy-aws-core/tests/unit/config/test_resolver.py +++ b/packages/smithy-aws-core/tests/unit/config/test_resolver.py @@ -59,7 +59,6 @@ async def test_defaults(self) -> None: ) assert config.disable_request_compression is False assert config.request_min_compression_size_bytes == 10240 - assert config.request_compression_encoding is None async def test_shared_profile_settings(self) -> None: fs = FakeFileSystem( @@ -109,11 +108,9 @@ async def test_environment_overrides_profile_and_explicit_overrides_environment( credentials_file_path="/fake/credentials", disable_request_compression=False, request_min_compression_size_bytes=0, - request_compression_encoding="gzip", ) assert explicit.disable_request_compression is False assert explicit.request_min_compression_size_bytes == 0 - assert explicit.request_compression_encoding == "gzip" @pytest.mark.parametrize("minimum", [-1, 10485761, True, 1.5, "large"]) async def test_invalid_threshold(self, minimum: Any) -> None: @@ -127,18 +124,8 @@ async def test_invalid_threshold(self, minimum: Any) -> None: request_min_compression_size_bytes=minimum, ) - async def test_explicit_gzip_conflicts_with_environment_disable(self) -> None: - with patch.dict( - os.environ, {"AWS_DISABLE_REQUEST_COMPRESSION": "true"}, clear=True - ): - with pytest.raises( - ConfigValidationError, match="disable_request_compression" - ): - await AsyncAwsConfig.resolve( - region="us-west-2", - fs=NullFileSystem(), - request_compression_encoding="gzip", - ) + def test_custom_compression_encoding_is_not_a_config_option(self) -> None: + assert "request_compression_encoding" not in AsyncAwsConfig.__dataclass_fields__ async def test_invalid_environment_threshold_is_rejected(self) -> None: with patch.dict( diff --git a/packages/smithy-http/src/smithy_http/compression.py b/packages/smithy-http/src/smithy_http/compression.py index 97c787d37..0564643f4 100644 --- a/packages/smithy-http/src/smithy_http/compression.py +++ b/packages/smithy-http/src/smithy_http/compression.py @@ -34,7 +34,6 @@ def from_operation( config = context.get("config") disabled = getattr(config, "disable_request_compression", False) minimum_size = getattr(config, "request_min_compression_size_bytes", 10240) - explicit = getattr(config, "request_compression_encoding", None) if not isinstance(disabled, bool): raise ValueError("disable_request_compression must be a boolean") if ( @@ -45,19 +44,9 @@ def from_operation( raise ValueError( "request_min_compression_size_bytes must be an integer between 0 and 10485760" ) - if explicit not in (None, "gzip"): - raise ValueError("request_compression_encoding must be None or 'gzip'") - if disabled and explicit is not None: - raise ValueError( - "request_compression_encoding conflicts with disable_request_compression" - ) trait = operation.schema.get_trait(RequestCompressionTrait) encodings = trait.encodings if trait is not None else [] supported = any(encoding.lower() == "gzip" for encoding in encodings) - if explicit is not None and not supported: - raise ValueError( - f"Operation {operation.schema.id.name} does not support gzip request compression" - ) if disabled or not supported: return cls() stream = operation.input_stream_member @@ -68,13 +57,9 @@ def from_operation( and stream.member_target.shape_type == ShapeType.UNION ) ) - if explicit is not None and unsafe_stream: - raise ValueError( - f"Operation {operation.schema.id.name} does not support gzip request compression" - ) if unsafe_stream: return cls() - return cls(encoding="gzip", minimum_size=0 if explicit else minimum_size) + return cls(encoding="gzip", minimum_size=minimum_size) def compress( self, body: StreamingBlob, fields: Fields, *, streaming: bool = False diff --git a/packages/smithy-http/tests/unit/aio/test_protocols.py b/packages/smithy-http/tests/unit/aio/test_protocols.py index e3008d67f..5fc13bdd7 100644 --- a/packages/smithy-http/tests/unit/aio/test_protocols.py +++ b/packages/smithy-http/tests/unit/aio/test_protocols.py @@ -195,7 +195,7 @@ def serialize_members(self, serializer: ShapeSerializer) -> None: input=OptionalPayloadInput(), endpoint=URI(host="example.com"), context=TypedProperties( - {"config": SimpleNamespace(request_compression_encoding="gzip")} + {"config": SimpleNamespace(request_min_compression_size_bytes=0)} ), ) assert request.fields["Content-Encoding"].as_string() == "gzip" @@ -204,10 +204,7 @@ def serialize_members(self, serializer: ShapeSerializer) -> None: assert int(request.fields["Content-Length"].as_string()) == len(wire) -@pytest.mark.parametrize("explicit", [None, "gzip"]) -async def test_http_binding_protocol_compresses_modeled_request( - explicit: str | None, -) -> None: +async def test_http_binding_protocol_compresses_modeled_request() -> None: operation = APIOperation( input=StructInput, output=MockOutput, @@ -231,9 +228,8 @@ async def test_http_binding_protocol_compresses_modeled_request( effective_auth_schemes=[], ) config = SimpleNamespace( - request_compression_encoding=explicit, disable_request_compression=False, - request_min_compression_size_bytes=0 if explicit is None else 10240, + request_min_compression_size_bytes=0, ) request = MockBindingProtocol().serialize_request( operation=operation, diff --git a/packages/smithy-http/tests/unit/test_compression.py b/packages/smithy-http/tests/unit/test_compression.py index 25056ed4e..e70ffd9be 100644 --- a/packages/smithy-http/tests/unit/test_compression.py +++ b/packages/smithy-http/tests/unit/test_compression.py @@ -27,7 +27,6 @@ def make_policy( *, encodings: list[str] | None = None, modeled: bool = True, - explicit: str | None = None, disabled: bool = False, threshold: int = 10240, stream: Schema | None = None, @@ -49,7 +48,6 @@ def make_policy( context = TypedProperties( { "config": SimpleNamespace( - request_compression_encoding=explicit, disable_request_compression=disabled, request_min_compression_size_bytes=threshold, ) @@ -63,12 +61,12 @@ def make_policy( @pytest.mark.parametrize( "payload", [b"", b"hello", bytearray(b"hello"), BytesIO(b"hello")] ) -async def test_explicit_gzip_round_trip_below_threshold( +async def test_zero_threshold_gzip_round_trip( payload: bytes | bytearray | BytesIO, ) -> None: expected = payload.getvalue() if isinstance(payload, BytesIO) else payload fields = Fields([Field(name="Content-Length", values=["999"])]) - result = make_policy(explicit="gzip").compress(payload, fields) + result = make_policy(threshold=0).compress(payload, fields) wire = await AsyncBytesReader(result).read() assert gzip.decompress(wire) == expected assert fields["Content-Encoding"].as_string() == "gzip" @@ -94,33 +92,22 @@ async def test_automatic_compression_threshold(size: int, compressed: bool) -> N @pytest.mark.parametrize( "kwargs", [{"modeled": False}, {"disabled": True}, {"encodings": ["unknown"]}] ) +@pytest.mark.parametrize("threshold", [0, 10240]) async def test_automatic_compression_skips_ineligible_request( kwargs: dict[str, Any], + threshold: int, ) -> None: payload = b"x" * 11000 fields = Fields() - result = make_policy(**kwargs).compress(payload, fields) + result = make_policy(threshold=threshold, **kwargs).compress(payload, fields) assert await AsyncBytesReader(result).read() == payload assert "Content-Encoding" not in fields -@pytest.mark.parametrize( - "kwargs", [{"modeled": False}, {"disabled": True}, {"encodings": ["unknown"]}] -) -def test_explicit_gzip_fails_instead_of_silent_fallback(kwargs: dict[str, Any]) -> None: - with pytest.raises(ValueError, match=r"gzip|compression"): - make_policy(explicit="gzip", **kwargs).compress(b"hello", Fields()) - - -def test_unknown_explicit_codec_is_rejected() -> None: - with pytest.raises(ValueError, match=r"zstd|encoding"): - make_policy(explicit="zstd").compress(b"hello", Fields()) - - async def test_gzip_appends_existing_content_encoding() -> None: fields = Fields([Field(name="Content-Encoding", values=["identity"])]) wire = await AsyncBytesReader( - make_policy(explicit="gzip").compress(b"hello", fields) + make_policy(threshold=0).compress(b"hello", fields) ).read() assert gzip.decompress(wire) == b"hello" assert "identity" in fields["Content-Encoding"].as_string() @@ -175,7 +162,7 @@ async def source() -> AsyncIterator[bytes]: @pytest.mark.parametrize("kind", ["requires_length", "eventstream"]) -async def test_incompatible_streams_are_skipped_or_rejected(kind: str) -> None: +async def test_incompatible_streams_are_skipped(kind: str) -> None: target = Schema( id=ShapeID("example#Stream"), shape_type=ShapeType.BLOB if kind == "requires_length" else ShapeType.UNION, @@ -191,5 +178,3 @@ async def test_incompatible_streams_are_skipped_or_rejected(kind: str) -> None: == body ) assert "Content-Encoding" not in fields - with pytest.raises(ValueError, match="gzip"): - make_policy(explicit="gzip", stream=stream) From 1ae921bac24a6ecebc61b8b43cd36382d359b4ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gabriel=20Silva=20Gouv=C3=AAa?= <64869705+gouveags@users.noreply.github.com> Date: Mon, 26 Oct 2026 10:30:46 -0300 Subject: [PATCH 5/5] Describe standard compression configuration in release note --- .../smithy-http-feature-dc6d4536b20e4253a86771e28a53dfd9.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/smithy-http/.changes/next-release/smithy-http-feature-dc6d4536b20e4253a86771e28a53dfd9.json b/packages/smithy-http/.changes/next-release/smithy-http-feature-dc6d4536b20e4253a86771e28a53dfd9.json index 57c16ab7f..a94a032d3 100644 --- a/packages/smithy-http/.changes/next-release/smithy-http-feature-dc6d4536b20e4253a86771e28a53dfd9.json +++ b/packages/smithy-http/.changes/next-release/smithy-http-feature-dc6d4536b20e4253a86771e28a53dfd9.json @@ -1,4 +1,4 @@ { "type": "feature", - "description": "Compress eligible HTTP request bodies with gzip before signing, with configurable thresholds, an explicit encoding choice, and streaming support that preserves seekable sources without buffering the entire compressed payload." -} \ No newline at end of file + "description": "Compress eligible HTTP request bodies with gzip before signing, with configurable thresholds and streaming support that preserves seekable sources without buffering the entire compressed payload." +}