test(extras): fix the #811 deploy test race

packages/aws-durable-execution-sdk-python-extras/README.md

@@ -124,7 +124,7 @@ A step retry costs one durable operation and one more Lambda invocation. Most fa
 1. **Inside one step attempt.** The boto3 client retries throttling and server errors of `RunMicrovm`, `CreateMicrovmAuthToken`, and `TerminateMicrovm`. The request step retries the POST itself: connection errors, timeouts, 429, and 5xx, with a delay from 250 milliseconds to 4 seconds. It retries one 401 or 403 with a new auth token. The endpoint answers this way until the MicroVM's `run` hook returns. This tier ends after `retry_window`, or 10 seconds before the invocation times out, whichever comes first.
 2. **The step retry strategy.** When the first tier gives up, the step retries after a longer delay, outside the invocation. `default_microvm_retry_strategy` is the default, unless the config sets `retry_strategy`. It retries throttling, server errors, connection failures, timeouts, and the end of the first tier, for at most 5 attempts, with a delay of 2, 4, 8, and 16 seconds. Every other error, such as `ValidationException`, `AccessDeniedException`, or `InsufficientCapacityException`, fails the step at once.
 
-A route that answers another 4xx status fails the operation without a retry, with `MicrovmDeliveryError`. So does a redirect, 3xx. The operation does not follow redirects, and the message names the `Location`. Set the path that the route serves, for example with its trailing slash. The JavaScript operation follows redirects.
+A route that answers another 4xx status fails the operation without a retry, with `MicrovmDeliveryError`. So does a redirect, 3xx. The operation does not follow redirects, and the message names the `Location`. Set the path that the route serves, for example with its trailing slash. The JavaScript operation also fails every 3xx at once, with `MicrovmDeliveryError`.
 
 The POST ignores the proxy settings in the environment, as the JavaScript SDK's `fetch` does. boto3 reads `HTTPS_PROXY`. So in a network that reaches the MicroVM endpoint only through a proxy, the launch succeeds and every POST fails. Pass an `opener` in the config for that network, for example `urllib.request.build_opener(urllib.request.ProxyHandler({"https": "http://proxy.internal:3128"}))`. When an opener follows a redirect, the delivery still fails at once with `MicrovmDeliveryError`. The followed request carries no auth token, because the operation adds its headers as unredirected headers.
 

packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py

@@ -273,14 +273,13 @@ def send_job(options: SendJobOptions) -> int:
     3. Any other 4xx, or a 3xx: raise :class:`MicrovmRequestRejectedError`
        at once.
 
-    The function does not follow a redirect. The JavaScript SDK's ``fetch``
-    follows one, and it differs in two ways. A 301, 302, or 303 turns the
-    POST into a GET without the job, and the GET's 2xx counts as accepted.
-    So the job never starts, and the operation waits until its timeout. A
-    307 or 308 to another host receives the auth token. So a redirect fails
-    the delivery at once, with the ``Location`` in the message. The headers
-    are added as unredirected headers, so an opener that follows a redirect
-    does not send the auth token on either.
+    The function does not follow a redirect, and neither does the JavaScript
+    operation. A followed 301, 302, or 303 would turn the POST into a GET
+    without the job. A followed 307 or 308 to another host would receive the
+    auth token. So a redirect fails the delivery at once, with the
+    ``Location`` in the message. The headers are added as unredirected
+    headers, so an opener that follows a redirect does not send the auth
+    token on either.
 
     The delay starts at 250 milliseconds and doubles up to 4 seconds. The
     tier ends at the retry window, or 10 seconds before the invocation times

packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py

@@ -1197,6 +1197,49 @@ def handler(event: Any, context: DurableContext) -> Any:
     return handler, invocations
 
 
+class _InvocationLog(list[Any]):
+    """A request list for the fake endpoint. It also records, for each
+    request, the number of the invocation that sent it."""
+
+    def __init__(self, invocations: list[int]) -> None:
+        super().__init__()
+        self._invocations = invocations
+        self.sent_in: list[int] = []
+
+    def append(self, request: Any) -> None:
+        # The request step blocks its invocation until the POST gets an
+        # answer or times out. The fake endpoint answers at once. The local
+        # runner runs one invocation at a time. So the invocation that sent
+        # the request is the last one that started.
+        self.sent_in.append(len(self._invocations))
+        super().append(request)
+
+
+def record_invocation_of_each_request(
+    endpoint: Any, invocations: list[int]
+) -> list[int]:
+    """Return a list that gets the invocation number of each request that
+    ``endpoint`` receives, in order. ``invocations`` is the counter of
+    :func:`handler_across_deploy`."""
+    log = _InvocationLog(invocations)
+    endpoint.requests = log
+    return log.sent_in
+
+
+def two_jobs_with_a_wait_between(
+    vm: MicrovmSession, ctx: DurableContext, event: Any
+) -> Any:
+    """Two jobs with a durable wait between them.
+
+    In the local runner, a durable wait ends the invocation. So the second
+    job is delivered in a later invocation than the first, after the deploy
+    of :func:`handler_across_deploy`."""
+    build = vm.invoke(ctx, "build", job_input=event, options=job("/build"))
+    ctx.wait(Duration.from_seconds(1), name="pause")
+    test = vm.invoke(ctx, "test", job_input=build, options=job())
+    return {"build": build, "test": test, "microvm": vm.microvm_id}
+
+
 def checkpoint_launch_as(monkeypatch: pytest.MonkeyPatch, **fields: Any) -> None:
     """Make the launch step checkpoint the MicroVM fields of a real launch,
     plus ``fields``, and no resume mode of its own. So a test controls the
@@ -1257,14 +1300,22 @@ def test_still_resumes_before_each_job_when_a_deploy_turns_suspending_off(
     client = fake_client(
         endpoint=endpoint.url, states=["RUNNING", "SUSPENDED", "RUNNING"]
     )
-    handler, invocations = handler_across_deploy(client, first, {}, two_jobs)
+    handler, invocations = handler_across_deploy(
+        client, first, {}, two_jobs_with_a_wait_between
+    )
+    sent_in = record_invocation_of_each_request(endpoint, invocations)
 
     result = run(handler, None, [("build", succeed_with(1)), ("test", succeed_with(2))])
 
     assert outcome(result)["result"]["test"] == 2
-    assert len(invocations) > 1
-    # The second job runs after the deploy. It checks the state, and resumes
-    # the MicroVM before delivery.
+    # The first job runs in the first invocation, with the launch's config.
+    # The second job runs in a later invocation, after the deploy.
+    assert len(sent_in) == 2
+    assert sent_in[0] == 1
+    assert sent_in[1] > sent_in[0]
+    # The first job checks the state once: RUNNING. The second job checks it
+    # after the deploy: SUSPENDED. It resumes the MicroVM, and checks the
+    # state again before delivery: RUNNING.
     assert len(client.requests("get_microvm")) == 3
     assert client.requests("resume_microvm") == [{"microvmIdentifier": "mvm-1"}]
     assert len(endpoint.requests) == 2
@@ -1280,15 +1331,21 @@ def test_makes_no_state_check_when_a_deploy_turns_suspending_on(
     endpoint = fake_endpoint()
     client = fake_client(endpoint=endpoint.url)
     handler, invocations = handler_across_deploy(
-        client, {}, {"auto_suspend_on_idle": True}, two_jobs
+        client, {}, {"auto_suspend_on_idle": True}, two_jobs_with_a_wait_between
     )
+    sent_in = record_invocation_of_each_request(endpoint, invocations)
 
     result = run(handler, None, [("build", succeed_with(1)), ("test", succeed_with(2))])
 
     assert outcome(result)["result"]["test"] == 2
-    assert len(invocations) > 1
+    # The first job runs in the first invocation, with the launch's config.
+    # The second job runs in a later invocation, after the deploy.
+    assert len(sent_in) == 2
+    assert sent_in[0] == 1
+    assert sent_in[1] > sent_in[0]
     [launch] = client.requests("run_microvm")
     assert "autoSuspendIdleSeconds" not in json.loads(launch["runHookPayload"])
+    # The second job runs after the deploy. It still checks no state.
     assert client.requests("get_microvm") == []
     assert len(endpoint.requests) == 2
     assert launch_checkpoint(result)["resumeBeforeDelivery"] is False