From c1064dfcf5af0b1d549f3ecec17e38d9ef7a4fb3 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Fri, 23 Oct 2026 10:18:54 -0700 Subject: [PATCH 1/4] test(extras): end the invocation between two jobs Two session tests of #811 check the resume mode across a deploy. The handler counts its invocations. The first invocation uses the launch's config. Every later invocation uses the deployed config. Each test runs two jobs and asserts len(invocations) > 1. 1. The run fixture completes each callback as soon as the job creates it. 2. A checkpoint response carries every operation that changed since the last response. 3. So the first callback's result can reach the first invocation. 4. So both jobs can be delivered in the first invocation, with the launch's config. 5. A suspension while the second job waits then satisfies len(invocations) > 1. No job is delivered after the deploy. 6. If both callbacks complete early, the execution ends in one invocation. The assertion then fails. The Codex review of #811 reported this race. In 65 local runs of the old test body, the second job was delivered in invocation 2. The test body did not ensure that order. The two tests now run a durable wait between the jobs. A durable wait always ends the invocation. The local runner skips the wait's time. So the second job is delivered in a later invocation, after the deploy, in every run. Each test also records the invocation that sent each request. It asserts that the first request was sent in invocation 1, and the second request in a later invocation. The tests fail against the session code before #811. The failures are at the GetMicrovm count: 1 instead of 3, and 1 instead of 0. --- .../tests/microvm/session_test.py | 66 +++++++++++++++++-- 1 file changed, 62 insertions(+), 4 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py index 0fa3f5c0e..1e7666a94 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py @@ -1197,6 +1197,48 @@ def handler(event: Any, context: DurableContext) -> Any: return handler, invocations +class _InvocationLog(list[Any]): + """A request list for the fake endpoint. It also records, for each + request, the number of the invocation that sent it.""" + + def __init__(self, invocations: list[int]) -> None: + super().__init__() + self._invocations = invocations + self.sent_in: list[int] = [] + + def append(self, request: Any) -> None: + # The request step blocks its invocation until the POST gets an + # answer. The local runner runs one invocation at a time. So the + # invocation that sent the request is the last one that started. + self.sent_in.append(len(self._invocations)) + super().append(request) + + +def record_invocation_of_each_request( + endpoint: Any, invocations: list[int] +) -> list[int]: + """Return a list that gets the invocation number of each request that + ``endpoint`` receives, in order. ``invocations`` is the counter of + :func:`handler_across_deploy`.""" + log = _InvocationLog(invocations) + endpoint.requests = log + return log.sent_in + + +def two_jobs_in_two_invocations( + vm: MicrovmSession, ctx: DurableContext, event: Any +) -> Any: + """Two jobs with a durable wait between them. + + A durable wait always ends the invocation. So the second job is delivered + in a later invocation than the first, after the deploy of + :func:`handler_across_deploy`.""" + build = vm.invoke(ctx, "build", job_input=event, options=job("/build")) + ctx.wait(Duration.from_seconds(1), name="pause") + test = vm.invoke(ctx, "test", job_input=build, options=job()) + return {"build": build, "test": test, "microvm": vm.microvm_id} + + def checkpoint_launch_as(monkeypatch: pytest.MonkeyPatch, **fields: Any) -> None: """Make the launch step checkpoint the MicroVM fields of a real launch, plus ``fields``, and no resume mode of its own. So a test controls the @@ -1257,14 +1299,23 @@ def test_still_resumes_before_each_job_when_a_deploy_turns_suspending_off( client = fake_client( endpoint=endpoint.url, states=["RUNNING", "SUSPENDED", "RUNNING"] ) - handler, invocations = handler_across_deploy(client, first, {}, two_jobs) + handler, invocations = handler_across_deploy( + client, first, {}, two_jobs_in_two_invocations + ) + sent_in = record_invocation_of_each_request(endpoint, invocations) result = run(handler, None, [("build", succeed_with(1)), ("test", succeed_with(2))]) assert outcome(result)["result"]["test"] == 2 assert len(invocations) > 1 - # The second job runs after the deploy. It checks the state, and resumes - # the MicroVM before delivery. + # The first job runs in the first invocation, with the launch's config. + # The second job runs in a later invocation, after the deploy. + assert len(sent_in) == 2 + assert sent_in[0] == 1 + assert sent_in[1] > sent_in[0] + # The first job checks the state once: RUNNING. The second job checks it + # after the deploy: SUSPENDED. It resumes the MicroVM, and checks the + # state again before delivery: RUNNING. assert len(client.requests("get_microvm")) == 3 assert client.requests("resume_microvm") == [{"microvmIdentifier": "mvm-1"}] assert len(endpoint.requests) == 2 @@ -1280,15 +1331,22 @@ def test_makes_no_state_check_when_a_deploy_turns_suspending_on( endpoint = fake_endpoint() client = fake_client(endpoint=endpoint.url) handler, invocations = handler_across_deploy( - client, {}, {"auto_suspend_on_idle": True}, two_jobs + client, {}, {"auto_suspend_on_idle": True}, two_jobs_in_two_invocations ) + sent_in = record_invocation_of_each_request(endpoint, invocations) result = run(handler, None, [("build", succeed_with(1)), ("test", succeed_with(2))]) assert outcome(result)["result"]["test"] == 2 assert len(invocations) > 1 + # The first job runs in the first invocation, with the launch's config. + # The second job runs in a later invocation, after the deploy. + assert len(sent_in) == 2 + assert sent_in[0] == 1 + assert sent_in[1] > sent_in[0] [launch] = client.requests("run_microvm") assert "autoSuspendIdleSeconds" not in json.loads(launch["runHookPayload"]) + # The second job runs after the deploy. It still checks no state. assert client.requests("get_microvm") == [] assert len(endpoint.requests) == 2 assert launch_checkpoint(result)["resumeBeforeDelivery"] is False From 087e39f2869b22d286db83e2fdeae52a95f6160e Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Fri, 23 Oct 2026 10:18:54 -0700 Subject: [PATCH 2/4] docs(extras): state the JS redirect behavior The README said that the JavaScript operation follows redirects. aws/aws-durable-execution-sdk-js#1036 changed that. The JavaScript request step now passes redirect: "manual". It fails every 3xx at once. The operation then raises MicrovmDeliveryError. So the README now states that the JavaScript operation also fails every 3xx at once, with MicrovmDeliveryError. --- packages/aws-durable-execution-sdk-python-extras/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/aws-durable-execution-sdk-python-extras/README.md b/packages/aws-durable-execution-sdk-python-extras/README.md index c30f30ce5..273cb3f33 100644 --- a/packages/aws-durable-execution-sdk-python-extras/README.md +++ b/packages/aws-durable-execution-sdk-python-extras/README.md @@ -124,7 +124,7 @@ A step retry costs one durable operation and one more Lambda invocation. Most fa 1. **Inside one step attempt.** The boto3 client retries throttling and server errors of `RunMicrovm`, `CreateMicrovmAuthToken`, and `TerminateMicrovm`. The request step retries the POST itself: connection errors, timeouts, 429, and 5xx, with a delay from 250 milliseconds to 4 seconds. It retries one 401 or 403 with a new auth token. The endpoint answers this way until the MicroVM's `run` hook returns. This tier ends after `retry_window`, or 10 seconds before the invocation times out, whichever comes first. 2. **The step retry strategy.** When the first tier gives up, the step retries after a longer delay, outside the invocation. `default_microvm_retry_strategy` is the default, unless the config sets `retry_strategy`. It retries throttling, server errors, connection failures, timeouts, and the end of the first tier, for at most 5 attempts, with a delay of 2, 4, 8, and 16 seconds. Every other error, such as `ValidationException`, `AccessDeniedException`, or `InsufficientCapacityException`, fails the step at once. -A route that answers another 4xx status fails the operation without a retry, with `MicrovmDeliveryError`. So does a redirect, 3xx. The operation does not follow redirects, and the message names the `Location`. Set the path that the route serves, for example with its trailing slash. The JavaScript operation follows redirects. +A route that answers another 4xx status fails the operation without a retry, with `MicrovmDeliveryError`. So does a redirect, 3xx. The operation does not follow redirects, and the message names the `Location`. Set the path that the route serves, for example with its trailing slash. The JavaScript operation also fails every 3xx at once, with `MicrovmDeliveryError`. The POST ignores the proxy settings in the environment, as the JavaScript SDK's `fetch` does. boto3 reads `HTTPS_PROXY`. So in a network that reaches the MicroVM endpoint only through a proxy, the launch succeeds and every POST fails. Pass an `opener` in the config for that network, for example `urllib.request.build_opener(urllib.request.ProxyHandler({"https": "http://proxy.internal:3128"}))`. When an opener follows a redirect, the delivery still fails at once with `MicrovmDeliveryError`. The followed request carries no auth token, because the operation adds its headers as unredirected headers. From 27f35bd80a593565f3b759e5d62d3b3e9f72f032 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Fri, 23 Oct 2026 12:21:28 -0700 Subject: [PATCH 3/4] docs(extras): fix the send_job redirect docstring The send_job docstring said that the JavaScript operation follows redirects. aws/aws-durable-execution-sdk-js#1036 changed that. The JavaScript request step now fails every 3xx at once. So the docstring contradicted the corrected README sentence. The docstring now states that neither operation follows a redirect. It then states why: a followed 301, 302, or 303 would send a GET without the job, and a followed 307 or 308 to another host would receive the auth token. --- .../microvm/request.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py index c2b98a197..c7f905c81 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py @@ -273,14 +273,13 @@ def send_job(options: SendJobOptions) -> int: 3. Any other 4xx, or a 3xx: raise :class:`MicrovmRequestRejectedError` at once. - The function does not follow a redirect. The JavaScript SDK's ``fetch`` - follows one, and it differs in two ways. A 301, 302, or 303 turns the - POST into a GET without the job, and the GET's 2xx counts as accepted. - So the job never starts, and the operation waits until its timeout. A - 307 or 308 to another host receives the auth token. So a redirect fails - the delivery at once, with the ``Location`` in the message. The headers - are added as unredirected headers, so an opener that follows a redirect - does not send the auth token on either. + The function does not follow a redirect, and neither does the JavaScript + operation. A followed 301, 302, or 303 would turn the POST into a GET + without the job. A followed 307 or 308 to another host would receive the + auth token. So a redirect fails the delivery at once, with the + ``Location`` in the message. The headers are added as unredirected + headers, so an opener that follows a redirect does not send the auth + token on either. The delay starts at 250 milliseconds and doubles up to 4 seconds. The tier ends at the retry window, or 10 seconds before the invocation times From 21ef3c211d1c7f222aa4aa9b04b7a42cc9fd69a5 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Fri, 23 Oct 2026 12:21:28 -0700 Subject: [PATCH 4/4] test(extras): name the wait between the two jobs The handler two_jobs_in_two_invocations runs two jobs with a durable wait between them. The execution runs 4 invocations, not 2. So the name is now two_jobs_with_a_wait_between. The SDK's wait returns without a suspension if the START checkpoint response marks the wait SUCCEEDED. The local runner always answers a START with STARTED. So the docstring now states that a durable wait ends the invocation in the local runner. A request can also time out without an answer. So the comment in _InvocationLog.append now says that the POST gets an answer or times out, and that the fake endpoint answers at once. The sent_in assertions require at least 2 invocations. So the two deploy tests no longer assert len(invocations) > 1. --- .../tests/microvm/session_test.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py index 1e7666a94..4d3c83a8e 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py @@ -1208,8 +1208,9 @@ def __init__(self, invocations: list[int]) -> None: def append(self, request: Any) -> None: # The request step blocks its invocation until the POST gets an - # answer. The local runner runs one invocation at a time. So the - # invocation that sent the request is the last one that started. + # answer or times out. The fake endpoint answers at once. The local + # runner runs one invocation at a time. So the invocation that sent + # the request is the last one that started. self.sent_in.append(len(self._invocations)) super().append(request) @@ -1225,14 +1226,14 @@ def record_invocation_of_each_request( return log.sent_in -def two_jobs_in_two_invocations( +def two_jobs_with_a_wait_between( vm: MicrovmSession, ctx: DurableContext, event: Any ) -> Any: """Two jobs with a durable wait between them. - A durable wait always ends the invocation. So the second job is delivered - in a later invocation than the first, after the deploy of - :func:`handler_across_deploy`.""" + In the local runner, a durable wait ends the invocation. So the second + job is delivered in a later invocation than the first, after the deploy + of :func:`handler_across_deploy`.""" build = vm.invoke(ctx, "build", job_input=event, options=job("/build")) ctx.wait(Duration.from_seconds(1), name="pause") test = vm.invoke(ctx, "test", job_input=build, options=job()) @@ -1300,14 +1301,13 @@ def test_still_resumes_before_each_job_when_a_deploy_turns_suspending_off( endpoint=endpoint.url, states=["RUNNING", "SUSPENDED", "RUNNING"] ) handler, invocations = handler_across_deploy( - client, first, {}, two_jobs_in_two_invocations + client, first, {}, two_jobs_with_a_wait_between ) sent_in = record_invocation_of_each_request(endpoint, invocations) result = run(handler, None, [("build", succeed_with(1)), ("test", succeed_with(2))]) assert outcome(result)["result"]["test"] == 2 - assert len(invocations) > 1 # The first job runs in the first invocation, with the launch's config. # The second job runs in a later invocation, after the deploy. assert len(sent_in) == 2 @@ -1331,14 +1331,13 @@ def test_makes_no_state_check_when_a_deploy_turns_suspending_on( endpoint = fake_endpoint() client = fake_client(endpoint=endpoint.url) handler, invocations = handler_across_deploy( - client, {}, {"auto_suspend_on_idle": True}, two_jobs_in_two_invocations + client, {}, {"auto_suspend_on_idle": True}, two_jobs_with_a_wait_between ) sent_in = record_invocation_of_each_request(endpoint, invocations) result = run(handler, None, [("build", succeed_with(1)), ("test", succeed_with(2))]) assert outcome(result)["result"]["test"] == 2 - assert len(invocations) > 1 # The first job runs in the first invocation, with the launch's config. # The second job runs in a later invocation, after the deploy. assert len(sent_in) == 2