feat(smithy-core): auto-fill idempotency tokens before serialization

codegen-output/aws-query/src/awsquery/models.pyi

@@ -341,6 +341,7 @@ class QueryIdempotencyTokenAutoFillInput(_SerdeShape):
     """Dataclass for QueryIdempotencyTokenAutoFillInput structure."""
 
     token: str | None = ...
+    """This field is autopopulated if not provided."""
 
 @dataclass(kw_only=True)
 class QueryIdempotencyTokenAutoFillOutput(_SerdeShape):

codegen-output/aws-query/tests/test_awsquery_protocol.py

@@ -1,6 +1,7 @@
 # Code generated by smithy-python-codegen DO NOT EDIT.
 
 from datetime import datetime, timezone
+from unittest.mock import patch
 from urllib.parse import parse_qsl
 
 from pytest import fail, mark, raises
@@ -1254,7 +1255,10 @@ async def test_sdk_appends_gzip_and_ignores_http_provided_encoding_aws_query_req
         )
 
 
-@mark.xfail()
+@patch(
+    "smithy_core._pipeline._default_idempotency_token",
+    new=lambda: "00000000-0000-4000-8000-000000000000",
+)
 async def test_query_protocol_idempotency_token_auto_fill_request_query_idempotency_token_auto_fill() -> (
     None
 ):
@@ -1335,6 +1339,10 @@ async def test_query_protocol_idempotency_token_auto_fill_request_query_idempote
         )
 
 
+@patch(
+    "smithy_core._pipeline._default_idempotency_token",
+    new=lambda: "00000000-0000-4000-8000-000000000000",
+)
 async def test_query_protocol_idempotency_token_auto_fill_is_set_request_query_idempotency_token_auto_fill() -> (
     None
 ):

codegen-output/ec2-query/src/ec2query/models.pyi

@@ -256,6 +256,7 @@ class QueryIdempotencyTokenAutoFillInput(_SerdeShape):
     """Dataclass for QueryIdempotencyTokenAutoFillInput structure."""
 
     token: str | None = ...
+    """This field is autopopulated if not provided."""
 
 @dataclass(kw_only=True)
 class QueryIdempotencyTokenAutoFillOutput(_SerdeShape):

codegen-output/ec2-query/tests/test_ec2query_protocol.py

@@ -1,6 +1,7 @@
 # Code generated by smithy-python-codegen DO NOT EDIT.
 
 from datetime import datetime, timezone
+from unittest.mock import patch
 from urllib.parse import parse_qsl
 
 from pytest import fail, mark, raises
@@ -989,7 +990,10 @@ async def test_sdk_appends_gzip_and_ignores_http_provided_encoding_ec2_query_req
         )
 
 
-@mark.xfail()
+@patch(
+    "smithy_core._pipeline._default_idempotency_token",
+    new=lambda: "00000000-0000-4000-8000-000000000000",
+)
 async def test_ec2_protocol_idempotency_token_auto_fill_request_query_idempotency_token_auto_fill() -> (
     None
 ):
@@ -1070,6 +1074,10 @@ async def test_ec2_protocol_idempotency_token_auto_fill_request_query_idempotenc
         )
 
 
+@patch(
+    "smithy_core._pipeline._default_idempotency_token",
+    new=lambda: "00000000-0000-4000-8000-000000000000",
+)
 async def test_ec2_protocol_idempotency_token_auto_fill_is_set_request_query_idempotency_token_auto_fill() -> (
     None
 ):

codegen-output/rest-json-1/src/restjson/_private/models_0.pyi

@@ -2255,6 +2255,7 @@ class QueryIdempotencyTokenAutoFillInput(_SerdeShape):
     """Dataclass for QueryIdempotencyTokenAutoFillInput structure."""
 
     token: str | None = ...
+    """This field is autopopulated if not provided."""
 
 @dataclass(kw_only=True)
 class QueryIdempotencyTokenAutoFillOutput(_SerdeShape):

codegen-output/rest-json-1/tests/test_restjson_protocol.py

@@ -2,6 +2,7 @@
 
 from datetime import datetime, timezone
 import json
+from unittest.mock import patch
 from urllib.parse import parse_qsl
 
 from pytest import fail, mark, raises
@@ -10936,7 +10937,10 @@ async def test_sdk_appended_gzip_after_provided_encoding_rest_json1_request_put_
         )
 
 
-@mark.xfail()
+@patch(
+    "smithy_core._pipeline._default_idempotency_token",
+    new=lambda: "00000000-0000-4000-8000-000000000000",
+)
 async def test_rest_json_query_idempotency_token_auto_fill_request_query_idempotency_token_auto_fill() -> (
     None
 ):
@@ -11011,6 +11015,10 @@ async def test_rest_json_query_idempotency_token_auto_fill_request_query_idempot
         )
 
 
+@patch(
+    "smithy_core._pipeline._default_idempotency_token",
+    new=lambda: "00000000-0000-4000-8000-000000000000",
+)
 async def test_rest_json_query_idempotency_token_auto_fill_is_set_request_query_idempotency_token_auto_fill() -> (
     None
 ):

codegen-output/rest-xml/src/restxml/models.pyi

@@ -942,6 +942,7 @@ class QueryIdempotencyTokenAutoFillInput(_SerdeShape):
     """Dataclass for QueryIdempotencyTokenAutoFillInput structure."""
 
     token: str | None = ...
+    """This field is autopopulated if not provided."""
 
 @dataclass(kw_only=True)
 class QueryIdempotencyTokenAutoFillOutput(_SerdeShape):

codegen-output/rest-xml/tests/test_restxml_protocol.py

@@ -1,6 +1,7 @@
 # Code generated by smithy-python-codegen DO NOT EDIT.
 
 from datetime import datetime, timezone
+from unittest.mock import patch
 from urllib.parse import parse_qsl
 
 from pytest import fail, mark, raises
@@ -5864,7 +5865,10 @@ async def test_sdk_appended_gzip_after_provided_encoding_rest_xml_request_put_wi
         )
 
 
-@mark.xfail()
+@patch(
+    "smithy_core._pipeline._default_idempotency_token",
+    new=lambda: "00000000-0000-4000-8000-000000000000",
+)
 async def test_query_idempotency_token_auto_fill_request_query_idempotency_token_auto_fill() -> (
     None
 ):
@@ -5939,6 +5943,10 @@ async def test_query_idempotency_token_auto_fill_request_query_idempotency_token
         )
 
 
+@patch(
+    "smithy_core._pipeline._default_idempotency_token",
+    new=lambda: "00000000-0000-4000-8000-000000000000",
+)
 async def test_query_idempotency_token_auto_fill_is_set_request_query_idempotency_token_auto_fill() -> (
     None
 ):

codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/AwsQueryProtocolGenerator.java

@@ -23,9 +23,6 @@ public final class AwsQueryProtocolGenerator implements ProtocolGenerator {
             "SDKAppliedContentEncoding_awsQuery",
             "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_awsQuery",
 
-            // TODO: support idempotency token autofill
-            "QueryProtocolIdempotencyTokenAutoFill",
-
             // TODO: support of the endpoint trait
             "AwsQueryEndpointTraitWithHostLabel",
             "AwsQueryEndpointTrait");

codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/Ec2QueryProtocolGenerator.java

@@ -23,9 +23,6 @@ public final class Ec2QueryProtocolGenerator implements ProtocolGenerator {
             "SDKAppliedContentEncoding_ec2Query",
             "SDKAppendsGzipAndIgnoresHttpProvidedEncoding_ec2Query",
 
-            // TODO: support idempotency token autofill
-            "Ec2ProtocolIdempotencyTokenAutoFill",
-
             // TODO: support of the endpoint trait
             "Ec2QueryEndpointTraitWithHostLabel",
             "Ec2QueryEndpointTrait");

codegen/aws/core/src/main/java/software/amazon/smithy/python/aws/codegen/RestXmlProtocolGenerator.java

@@ -25,9 +25,6 @@ public final class RestXmlProtocolGenerator implements ProtocolGenerator {
             "SDKAppliedContentEncoding_restXml",
             "SDKAppendedGzipAfterProvidedEncoding_restXml",
 
-            // TODO: support idempotency token autofill
-            "QueryIdempotencyTokenAutoFill",
-
             // TODO: support of the endpoint trait
             "RestXmlEndpointTraitWithHostLabelAndHttpBinding",
             "RestXmlEndpointTraitWithHostLabel",

codegen/core/src/main/java/software/amazon/smithy/python/codegen/CodegenUtils.java

@@ -29,11 +29,14 @@
 import software.amazon.smithy.codegen.core.Symbol;
 import software.amazon.smithy.model.Model;
 import software.amazon.smithy.model.knowledge.NullableIndex;
+import software.amazon.smithy.model.knowledge.OperationIndex;
 import software.amazon.smithy.model.node.Node;
 import software.amazon.smithy.model.shapes.MemberShape;
 import software.amazon.smithy.model.shapes.Shape;
 import software.amazon.smithy.model.traits.DefaultTrait;
+import software.amazon.smithy.model.traits.DocumentationTrait;
 import software.amazon.smithy.model.traits.ErrorTrait;
+import software.amazon.smithy.model.traits.IdempotencyTokenTrait;
 import software.amazon.smithy.model.traits.StreamingTrait;
 import software.amazon.smithy.model.traits.TimestampFormatTrait;
 import software.amazon.smithy.model.traits.TimestampFormatTrait.Format;
@@ -179,6 +182,33 @@ public static boolean isEventStreamMember(Model model, MemberShape member) {
         return target.isUnionShape() && target.hasTrait(StreamingTrait.class);
     }
 
+    /**
+     * The note appended to the docs of an idempotency token member the client fills.
+     * Matches botocore's wording.
+     */
+    public static final String IDEMPOTENCY_TOKEN_DOCS = "This field is autopopulated if not provided.";
+
+    /**
+     * Gets the documentation to generate for a member.
+     *
+     * <p>This is the member's documentation trait (or its target's). Members the client
+     * auto-fills, {@code @idempotencyToken} members of an operation input, also get
+     * {@link #IDEMPOTENCY_TOKEN_DOCS} as a final paragraph, or as their only docs.
+     *
+     * @param model The model being generated.
+     * @param member The member to document.
+     * @return Returns the member's documentation, if it has any.
+     */
+    public static Optional<String> getMemberDocs(Model model, MemberShape member) {
+        var docs = member.getMemberTrait(model, DocumentationTrait.class).map(DocumentationTrait::getValue);
+        if (!member.hasTrait(IdempotencyTokenTrait.class)
+                || !OperationIndex.of(model).isInputStructure(member.getContainer())) {
+            return docs;
+        }
+        return Optional.of(docs.map(value -> value + "\n\n" + IDEMPOTENCY_TOKEN_DOCS)
+                .orElse(IDEMPOTENCY_TOKEN_DOCS));
+    }
+
     /**
      * Determines whether a member is required in the generated Python constructor — that is,
      * neither nullable nor carrying a default value.

codegen/core/src/main/java/software/amazon/smithy/python/codegen/HttpProtocolTestGenerator.java

@@ -34,6 +34,7 @@
 import software.amazon.smithy.model.shapes.Shape;
 import software.amazon.smithy.model.shapes.ShapeId;
 import software.amazon.smithy.model.shapes.StructureShape;
+import software.amazon.smithy.model.traits.IdempotencyTokenTrait;
 import software.amazon.smithy.model.traits.StreamingTrait;
 import software.amazon.smithy.model.traits.TimestampFormatTrait.Format;
 import software.amazon.smithy.protocoltests.traits.AppliesTo;
@@ -70,6 +71,10 @@ public final class HttpProtocolTestGenerator implements Runnable {
     private static final Symbol TEST_HTTP_SERVICE_ERR_SYMBOL = Symbol.builder()
             .name("TestHttpServiceError")
             .build();
+    /** The idempotency token value protocol tests expect when the caller leaves the member unset. */
+    private static final String TEST_IDEMPOTENCY_TOKEN = "00000000-0000-4000-8000-000000000000";
+    /** The runtime function that generates auto-filled idempotency tokens. */
+    private static final String IDEMPOTENCY_TOKEN_GENERATOR = "smithy_core._pipeline._default_idempotency_token";
 
     private final PythonSettings settings;
     private final Model model;
@@ -160,6 +165,7 @@ private void generateRequestTest(OperationShape operation, HttpRequestTestCase t
                 testCase,
                 String.format("%s_request_%s", testCase.getId(), operation.getId().getName()),
                 testFilter.test(operation, testCase),
+                () -> writeIdempotencyTokenPatch(operation),
                 () -> {
                     var hostSplit = testCase.getHost().orElse("example.com").split("/", 2);
                     var host = hostSplit[0];
@@ -634,6 +640,16 @@ private void writeTestBlock(
             String testName,
             boolean shouldSkip,
             Runnable f
+    ) {
+        writeTestBlock(testCase, testName, shouldSkip, () -> {}, f);
+    }
+
+    private void writeTestBlock(
+            HttpMessageTestCase testCase,
+            String testName,
+            boolean shouldSkip,
+            Runnable decorators,
+            Runnable f
     ) {
         LOGGER.fine(String.format("Writing test block for %s", testName));
         writer.addDependency(SmithyPythonDependency.PYTEST);
@@ -644,6 +660,7 @@ private void writeTestBlock(
             writer.addImport(SmithyPythonDependency.PYTEST.packageName(), "mark", "mark");
             writer.write("@mark.xfail()");
         }
+        decorators.run();
         writer.openBlock("async def test_$L() -> None:", "", CaseUtils.toSnakeCase(testName), () -> {
             testCase.getDocumentation().ifPresent(docs -> writer.writeDocs(docs, context));
             f.run();
@@ -690,6 +707,20 @@ private void writeSigV4TestConfig() {
                 """, RuntimeTypes.STATIC_CREDENTIALS_RESOLVER);
     }
 
+    /**
+     * Pins the auto-filled idempotency token so request tests can assert it.
+     * The generator is private to the runtime, so the test patches it. Only
+     * written for operations whose input has an {@code @idempotencyToken} member.
+     */
+    private void writeIdempotencyTokenPatch(OperationShape operation) {
+        var inputShape = model.expectShape(operation.getInputShape(), StructureShape.class);
+        if (inputShape.members().stream().noneMatch(member -> member.hasTrait(IdempotencyTokenTrait.class))) {
+            return;
+        }
+        writer.addStdlibImport("unittest.mock", "patch");
+        writer.write("@patch($S, new=lambda: $S)", IDEMPOTENCY_TOKEN_GENERATOR, TEST_IDEMPOTENCY_TOKEN);
+    }
+
     private void writeUtilStubs(Symbol serviceSymbol) {
         LOGGER.fine(String.format("Writing utility stubs for %s : %s", serviceSymbol.getName(), protocol.getName()));
         writer.addLocallyDefinedSymbol(TEST_HTTP_SERVICE_ERR_SYMBOL);

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/ModelStubGenerator.java

@@ -267,8 +267,7 @@ private void writeShapeDocs(PythonWriter writer, Shape shape, String fallback) {
     }
 
     private void writeMemberDocs(PythonWriter writer, MemberShape member) {
-        member.getMemberTrait(model, DocumentationTrait.class)
-                .ifPresent(trait -> writer.writeDocs(trait.getValue(), context));
+        CodegenUtils.getMemberDocs(model, member).ifPresent(docs -> writer.writeDocs(docs, context));
     }
 
     private boolean carriesResponseMetadata(Shape shape) {

codegen/core/src/main/java/software/amazon/smithy/python/codegen/generators/StructureGenerator.java

@@ -294,9 +294,7 @@ private void writeProperties() {
     }
 
     private void writeMemberDocs(MemberShape member) {
-        member.getMemberTrait(model, DocumentationTrait.class).ifPresent(trait -> {
-            writer.writeDocs(trait.getValue(), context);
-        });
+        CodegenUtils.getMemberDocs(model, member).ifPresent(docs -> writer.writeDocs(docs, context));
     }
 
     private List<MemberShape> filterPropertyMembers(List<MemberShape> members) {

codegen/core/src/main/java/software/amazon/smithy/python/codegen/integrations/RestJsonProtocolGenerator.java

@@ -31,9 +31,6 @@
 public class RestJsonProtocolGenerator implements ProtocolGenerator {
 
     private static final Set<String> TESTS_TO_SKIP = Set.of(
-            // This requires support of idempotency autofill
-            "RestJsonQueryIdempotencyTokenAutoFill",
-
             // This requires support of the httpChecksumRequired trait
             "RestJsonHttpChecksumRequired",