fix: derive transition backend from capture

src/commands/interaction/runtime/settle-transition-baseline.test.ts

@@ -0,0 +1,118 @@
+import assert from 'node:assert/strict';
+import { test } from 'vitest';
+import type { AgentDeviceBackend } from '../../../backend.ts';
+import { createLocalArtifactAdapter } from '../../../io.ts';
+import {
+  createAgentDevice,
+  createMemorySessionStore,
+  localCommandPolicy,
+} from '../../../runtime.ts';
+import {
+  elementSettledRoomSnapshot,
+  elementThreadsNoticeSnapshot,
+  elementTransientRoomSnapshot,
+} from './stable-capture.fixtures.ts';
+import { settleAfterInteraction } from './settle.ts';
+
+test('settle recovers the session baseline when resolved target evidence is absent', async () => {
+  let elapsedMs = 0;
+  const runtime = createAgentDevice({
+    backend: {
+      platform: 'ios',
+      captureSnapshot: async () => ({
+        snapshot: elapsedMs < 1_200 ? elementTransientRoomSnapshot : elementSettledRoomSnapshot,
+      }),
+    } satisfies AgentDeviceBackend,
+    artifacts: createLocalArtifactAdapter(),
+    sessions: createMemorySessionStore([
+      {
+        name: 'default',
+        snapshot: elementThreadsNoticeSnapshot,
+        refFrameSnapshot: elementThreadsNoticeSnapshot,
+      },
+    ]),
+    policy: localCommandPolicy(),
+    clock: {
+      now: () => elapsedMs,
+      sleep: async (ms: number) => {
+        elapsedMs += ms;
+      },
+    },
+  });
+
+  const outcome = await settleAfterInteraction(
+    runtime,
+    { session: 'default' },
+    {
+      resolved: {
+        kind: 'ref',
+        point: { x: 201, y: 795 },
+        target: { kind: 'ref', ref: '@e5' },
+      },
+      quietMs: 500,
+      timeoutMs: 5_000,
+    },
+  );
+
+  assert.equal(outcome.observation.settled, true);
+  assert.equal(
+    outcome.settledNodes?.some((node) => node.label === 'action file'),
+    false,
+  );
+  assert.ok(
+    outcome.observation.waitedMs >= 1_500,
+    `settled without recovered baseline after ${outcome.observation.waitedMs}ms`,
+  );
+});
+
+test('settle uses the authorized ref frame instead of a polluted evidence capture', async () => {
+  let elapsedMs = 0;
+  const runtime = createAgentDevice({
+    backend: {
+      platform: 'ios',
+      captureSnapshot: async () => ({
+        snapshot: elapsedMs < 1_200 ? elementTransientRoomSnapshot : elementSettledRoomSnapshot,
+      }),
+    } satisfies AgentDeviceBackend,
+    artifacts: createLocalArtifactAdapter(),
+    sessions: createMemorySessionStore([
+      {
+        name: 'default',
+        snapshot: elementThreadsNoticeSnapshot,
+        refFrameSnapshot: elementThreadsNoticeSnapshot,
+      },
+    ]),
+    policy: localCommandPolicy(),
+    clock: {
+      now: () => elapsedMs,
+      sleep: async (ms: number) => {
+        elapsedMs += ms;
+      },
+    },
+  });
+
+  const outcome = await settleAfterInteraction(
+    runtime,
+    { session: 'default' },
+    {
+      resolved: {
+        kind: 'ref',
+        point: { x: 201, y: 795 },
+        target: { kind: 'ref', ref: '@e5' },
+        preActionNodes: elementTransientRoomSnapshot.nodes,
+      },
+      quietMs: 500,
+      timeoutMs: 5_000,
+    },
+  );
+
+  assert.equal(outcome.observation.settled, true);
+  assert.equal(
+    outcome.settledNodes?.some((node) => node.label === 'action file'),
+    false,
+  );
+  assert.ok(
+    outcome.observation.waitedMs >= 1_500,
+    `settled against polluted evidence after ${outcome.observation.waitedMs}ms`,
+  );
+});

src/commands/interaction/runtime/settle.test.ts

@@ -171,7 +171,7 @@ test('never-settling content returns settled: false without an actionable diff',
 
 test('private-ax recovery resets the settle budget once', async () => {
   const before = buttonSnapshot();
-  const recoveredAfter = welcomeSnapshot();
+  const recoveredAfter = buttonSnapshot();
   recoveredAfter.snapshotQuality = {
     state: 'recovered',
     backend: 'private-ax',

src/commands/interaction/runtime/settle.ts

@@ -73,7 +73,7 @@ export async function settleAfterInteraction(
 ): Promise<SettleOutcome> {
   return await settleAfterAction(runtime, options, {
     ...params,
-    baselineNodes: resolveBaselineNodes(params.resolved),
+    baselineNodes: await resolveBaselineNodes(runtime, options, params.resolved),
     actionPoint: params.resolved.point,
   });
 }
@@ -202,8 +202,45 @@ export function settleEvidence(
   return { ...after, changedFromBefore };
 }
 
-function resolveBaselineNodes(resolved: ResolvedInteractionTarget): SnapshotNode[] {
-  return 'preActionNodes' in resolved && resolved.preActionNodes ? resolved.preActionNodes : [];
+async function resolveBaselineNodes(
+  runtime: AgentDeviceRuntime,
+  options: CommandContext,
+  resolved: ResolvedInteractionTarget,
+): Promise<SnapshotNode[]> {
+  const session = await runtime.sessions.get(options.session ?? 'default');
+  // A ref is authorized against the stored ref frame. Keep that visible presentation as the
+  // transition baseline: a best-effort evidence recapture can recover through private AX and see
+  // covered background controls that were not actionable when the ref was issued.
+  // Resolved-target evidence is best-effort at the contracts boundary. The session still owns the
+  // authoritative ref frame, so reuse it rather than silently turning a missing optional field
+  // into an empty transition/diff baseline. Fall back to the latest observation for point targets
+  // and pre-frame sessions.
+  return (
+    authorizedRefBaseline(resolved, session) ??
+    evidenceBaseline(resolved) ??
+    sessionBaseline(session)
+  );
+}
+
+function authorizedRefBaseline(
+  resolved: ResolvedInteractionTarget,
+  session: CommandSessionRecord | undefined,
+): SnapshotNode[] | undefined {
+  if (resolved.kind !== 'ref') return undefined;
+  return nonEmptyNodes(session?.refFrameSnapshot?.nodes);
+}
+
+function evidenceBaseline(resolved: ResolvedInteractionTarget): SnapshotNode[] | undefined {
+  if (!('preActionNodes' in resolved)) return undefined;
+  return nonEmptyNodes(resolved.preActionNodes);
+}
+
+function sessionBaseline(session: CommandSessionRecord | undefined): SnapshotNode[] {
+  return session?.refFrameSnapshot?.nodes ?? session?.snapshot?.nodes ?? [];
+}
+
+function nonEmptyNodes(nodes: SnapshotNode[] | undefined): SnapshotNode[] | undefined {
+  return nodes?.length ? nodes : undefined;
 }
 
 function buildSettleDiff(

src/commands/interaction/runtime/stable-capture.test.ts

@@ -114,7 +114,7 @@ test('settle confirms a broad replacement that begins after the first post-actio
 
 test('settle confirms against the pre-action tree when the stored snapshot already advanced', async () => {
   const { runtime } = transitionRuntime({
-    sessionSnapshot: elementTransientRoomSnapshot,
+    sessionSnapshot: { ...elementTransientRoomSnapshot, backend: undefined },
     settledAtMs: 1_200,
   });
 
@@ -135,6 +135,74 @@ test('settle confirms against the pre-action tree when the stored snapshot alrea
   );
 });
 
+test('settle confirms an iOS broad replacement when capture omits snapshot backend provenance', async () => {
+  const { runtime } = transitionRuntime({
+    omitSnapshotBackend: true,
+    settledAtMs: 1_200,
+  });
+
+  const outcome = await runStableCaptureLoop(
+    runtime,
+    { session: 'default' },
+    {
+      ...BROAD_TRANSITION_PARAMS,
+      // Presented iOS modals can be complete interaction captures without retaining the
+      // Application root in their projection.
+      broadTransitionBaselineNodes: elementThreadsNoticeSnapshot.nodes.slice(1),
+    },
+  );
+
+  assert.equal(outcome.settled, true);
+  assert.equal(
+    outcome.lastCapture?.snapshot.nodes.some((node) => node.label === 'action file'),
+    false,
+  );
+  assert.ok(
+    outcome.waitedMs >= 1_500,
+    `settled backend-less transition after ${outcome.waitedMs}ms`,
+  );
+});
+
+test('settle confirms a tiny modal replacement through a partial post-action projection', async () => {
+  const { runtime } = transitionRuntime({ omitViewportRoot: true, settledAtMs: 1_200 });
+
+  const outcome = await runStableCaptureLoop(
+    runtime,
+    { session: 'default' },
+    BROAD_TRANSITION_PARAMS,
+  );
+
+  assert.equal(outcome.settled, true);
+  assert.equal(
+    outcome.lastCapture?.snapshot.nodes.some((node) => node.label === 'action file'),
+    false,
+  );
+  assert.ok(
+    outcome.waitedMs >= 1_500,
+    `settled partial transitional tree after ${outcome.waitedMs}ms`,
+  );
+});
+
+test('settle keeps the default quiet window for a larger partial projection', async () => {
+  const { runtime } = transitionRuntime({ omitViewportRoot: true, settledAtMs: 1_200 });
+
+  const outcome = await runStableCaptureLoop(
+    runtime,
+    { session: 'default' },
+    {
+      ...BROAD_TRANSITION_PARAMS,
+      broadTransitionBaselineNodes: elementSettingsSnapshot([1_138, 1_423, 2_144, 2_434]).nodes,
+    },
+  );
+
+  assert.equal(outcome.settled, true);
+  assert.equal(
+    outcome.lastCapture?.snapshot.nodes.some((node) => node.label === 'action file'),
+    true,
+  );
+  assert.ok(outcome.waitedMs < 800, `partial projection settled after ${outcome.waitedMs}ms`);
+});
+
 test('settle honors an explicitly shorter quiet window across a broad replacement', async () => {
   const { runtime } = transitionRuntime();
 
@@ -149,14 +217,14 @@ test('settle honors an explicitly shorter quiet window across a broad replacemen
   assert.ok(outcome.waitedMs < 800, `short quiet window settled after ${outcome.waitedMs}ms`);
 });
 
-test('settle keeps the default quiet window for an overlapping local mutation', async () => {
+test('settle keeps the default quiet window for a backend-less overlapping local mutation', async () => {
   const localMutation = {
     ...elementSettledRoomSnapshot,
     nodes: elementSettledRoomSnapshot.nodes.map((node) =>
       node.label === 'Upload' ? { ...node, label: 'Add attachment' } : node,
     ),
   };
-  const { runtime } = staticSnapshotRuntime(localMutation);
+  const { runtime } = staticSnapshotRuntime(withoutSnapshotBackend(localMutation, true));
 
   const outcome = await runStableCaptureLoop(
     runtime,
@@ -197,6 +265,8 @@ function transitionRuntime(
   options: {
     captureBackend?: 'tree' | 'private-ax';
     firstCaptureKeepsBaseline?: boolean;
+    omitSnapshotBackend?: boolean;
+    omitViewportRoot?: boolean;
     settledAtMs?: number;
     sessionSnapshot?: SnapshotState;
   } = {},
@@ -217,13 +287,19 @@ function transitionRuntime(
       captureSnapshot: async () => {
         const keepsBaseline = options.firstCaptureKeepsBaseline === true && captures === 0;
         captures += 1;
+        const snapshot = keepsBaseline
+          ? elementThreadsNoticeSnapshot
+          : withCaptureBackend(
+              elapsedMs < settledAtMs ? elementTransientRoomSnapshot : elementSettledRoomSnapshot,
+              captureBackend,
+            );
         return {
-          snapshot: keepsBaseline
-            ? elementThreadsNoticeSnapshot
-            : withCaptureBackend(
-                elapsedMs < settledAtMs ? elementTransientRoomSnapshot : elementSettledRoomSnapshot,
-                captureBackend,
-              ),
+          snapshot: withoutSnapshotBackend(
+            options.omitViewportRoot === true
+              ? { ...snapshot, nodes: snapshot.nodes.slice(1) }
+              : snapshot,
+            options.omitSnapshotBackend === true,
+          ),
         };
       },
     } satisfies AgentDeviceBackend,
@@ -237,6 +313,12 @@ function transitionRuntime(
   return { runtime };
 }
 
+function withoutSnapshotBackend(snapshot: SnapshotState, omit: boolean): SnapshotState {
+  if (!omit) return snapshot;
+  const { backend: _backend, ...backendless } = snapshot;
+  return backendless;
+}
+
 function withCaptureBackend(
   snapshot: SnapshotState,
   backend: 'tree' | 'private-ax',