diff --git a/src/commands/interaction/runtime/settle-transition-baseline.test.ts b/src/commands/interaction/runtime/settle-transition-baseline.test.ts new file mode 100644 index 000000000..e6401d26e --- /dev/null +++ b/src/commands/interaction/runtime/settle-transition-baseline.test.ts @@ -0,0 +1,118 @@ +import assert from 'node:assert/strict'; +import { test } from 'vitest'; +import type { AgentDeviceBackend } from '../../../backend.ts'; +import { createLocalArtifactAdapter } from '../../../io.ts'; +import { + createAgentDevice, + createMemorySessionStore, + localCommandPolicy, +} from '../../../runtime.ts'; +import { + elementSettledRoomSnapshot, + elementThreadsNoticeSnapshot, + elementTransientRoomSnapshot, +} from './stable-capture.fixtures.ts'; +import { settleAfterInteraction } from './settle.ts'; + +test('settle recovers the session baseline when resolved target evidence is absent', async () => { + let elapsedMs = 0; + const runtime = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ + snapshot: elapsedMs < 1_200 ? elementTransientRoomSnapshot : elementSettledRoomSnapshot, + }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([ + { + name: 'default', + snapshot: elementThreadsNoticeSnapshot, + refFrameSnapshot: elementThreadsNoticeSnapshot, + }, + ]), + policy: localCommandPolicy(), + clock: { + now: () => elapsedMs, + sleep: async (ms: number) => { + elapsedMs += ms; + }, + }, + }); + + const outcome = await settleAfterInteraction( + runtime, + { session: 'default' }, + { + resolved: { + kind: 'ref', + point: { x: 201, y: 795 }, + target: { kind: 'ref', ref: '@e5' }, + }, + quietMs: 500, + timeoutMs: 5_000, + }, + ); + + assert.equal(outcome.observation.settled, true); + assert.equal( + outcome.settledNodes?.some((node) => node.label === 'action file'), + false, + ); + assert.ok( + outcome.observation.waitedMs >= 1_500, + `settled without recovered baseline after ${outcome.observation.waitedMs}ms`, + ); +}); + +test('settle uses the authorized ref frame instead of a polluted evidence capture', async () => { + let elapsedMs = 0; + const runtime = createAgentDevice({ + backend: { + platform: 'ios', + captureSnapshot: async () => ({ + snapshot: elapsedMs < 1_200 ? elementTransientRoomSnapshot : elementSettledRoomSnapshot, + }), + } satisfies AgentDeviceBackend, + artifacts: createLocalArtifactAdapter(), + sessions: createMemorySessionStore([ + { + name: 'default', + snapshot: elementThreadsNoticeSnapshot, + refFrameSnapshot: elementThreadsNoticeSnapshot, + }, + ]), + policy: localCommandPolicy(), + clock: { + now: () => elapsedMs, + sleep: async (ms: number) => { + elapsedMs += ms; + }, + }, + }); + + const outcome = await settleAfterInteraction( + runtime, + { session: 'default' }, + { + resolved: { + kind: 'ref', + point: { x: 201, y: 795 }, + target: { kind: 'ref', ref: '@e5' }, + preActionNodes: elementTransientRoomSnapshot.nodes, + }, + quietMs: 500, + timeoutMs: 5_000, + }, + ); + + assert.equal(outcome.observation.settled, true); + assert.equal( + outcome.settledNodes?.some((node) => node.label === 'action file'), + false, + ); + assert.ok( + outcome.observation.waitedMs >= 1_500, + `settled against polluted evidence after ${outcome.observation.waitedMs}ms`, + ); +}); diff --git a/src/commands/interaction/runtime/settle.test.ts b/src/commands/interaction/runtime/settle.test.ts index 06f69c506..9788502dc 100644 --- a/src/commands/interaction/runtime/settle.test.ts +++ b/src/commands/interaction/runtime/settle.test.ts @@ -171,7 +171,7 @@ test('never-settling content returns settled: false without an actionable diff', test('private-ax recovery resets the settle budget once', async () => { const before = buttonSnapshot(); - const recoveredAfter = welcomeSnapshot(); + const recoveredAfter = buttonSnapshot(); recoveredAfter.snapshotQuality = { state: 'recovered', backend: 'private-ax', diff --git a/src/commands/interaction/runtime/settle.ts b/src/commands/interaction/runtime/settle.ts index 4d56f0021..b90327edc 100644 --- a/src/commands/interaction/runtime/settle.ts +++ b/src/commands/interaction/runtime/settle.ts @@ -73,7 +73,7 @@ export async function settleAfterInteraction( ): Promise { return await settleAfterAction(runtime, options, { ...params, - baselineNodes: resolveBaselineNodes(params.resolved), + baselineNodes: await resolveBaselineNodes(runtime, options, params.resolved), actionPoint: params.resolved.point, }); } @@ -202,8 +202,45 @@ export function settleEvidence( return { ...after, changedFromBefore }; } -function resolveBaselineNodes(resolved: ResolvedInteractionTarget): SnapshotNode[] { - return 'preActionNodes' in resolved && resolved.preActionNodes ? resolved.preActionNodes : []; +async function resolveBaselineNodes( + runtime: AgentDeviceRuntime, + options: CommandContext, + resolved: ResolvedInteractionTarget, +): Promise { + const session = await runtime.sessions.get(options.session ?? 'default'); + // A ref is authorized against the stored ref frame. Keep that visible presentation as the + // transition baseline: a best-effort evidence recapture can recover through private AX and see + // covered background controls that were not actionable when the ref was issued. + // Resolved-target evidence is best-effort at the contracts boundary. The session still owns the + // authoritative ref frame, so reuse it rather than silently turning a missing optional field + // into an empty transition/diff baseline. Fall back to the latest observation for point targets + // and pre-frame sessions. + return ( + authorizedRefBaseline(resolved, session) ?? + evidenceBaseline(resolved) ?? + sessionBaseline(session) + ); +} + +function authorizedRefBaseline( + resolved: ResolvedInteractionTarget, + session: CommandSessionRecord | undefined, +): SnapshotNode[] | undefined { + if (resolved.kind !== 'ref') return undefined; + return nonEmptyNodes(session?.refFrameSnapshot?.nodes); +} + +function evidenceBaseline(resolved: ResolvedInteractionTarget): SnapshotNode[] | undefined { + if (!('preActionNodes' in resolved)) return undefined; + return nonEmptyNodes(resolved.preActionNodes); +} + +function sessionBaseline(session: CommandSessionRecord | undefined): SnapshotNode[] { + return session?.refFrameSnapshot?.nodes ?? session?.snapshot?.nodes ?? []; +} + +function nonEmptyNodes(nodes: SnapshotNode[] | undefined): SnapshotNode[] | undefined { + return nodes?.length ? nodes : undefined; } function buildSettleDiff( diff --git a/src/commands/interaction/runtime/stable-capture.test.ts b/src/commands/interaction/runtime/stable-capture.test.ts index 323818966..5f252d6bc 100644 --- a/src/commands/interaction/runtime/stable-capture.test.ts +++ b/src/commands/interaction/runtime/stable-capture.test.ts @@ -114,7 +114,7 @@ test('settle confirms a broad replacement that begins after the first post-actio test('settle confirms against the pre-action tree when the stored snapshot already advanced', async () => { const { runtime } = transitionRuntime({ - sessionSnapshot: elementTransientRoomSnapshot, + sessionSnapshot: { ...elementTransientRoomSnapshot, backend: undefined }, settledAtMs: 1_200, }); @@ -135,6 +135,74 @@ test('settle confirms against the pre-action tree when the stored snapshot alrea ); }); +test('settle confirms an iOS broad replacement when capture omits snapshot backend provenance', async () => { + const { runtime } = transitionRuntime({ + omitSnapshotBackend: true, + settledAtMs: 1_200, + }); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + ...BROAD_TRANSITION_PARAMS, + // Presented iOS modals can be complete interaction captures without retaining the + // Application root in their projection. + broadTransitionBaselineNodes: elementThreadsNoticeSnapshot.nodes.slice(1), + }, + ); + + assert.equal(outcome.settled, true); + assert.equal( + outcome.lastCapture?.snapshot.nodes.some((node) => node.label === 'action file'), + false, + ); + assert.ok( + outcome.waitedMs >= 1_500, + `settled backend-less transition after ${outcome.waitedMs}ms`, + ); +}); + +test('settle confirms a tiny modal replacement through a partial post-action projection', async () => { + const { runtime } = transitionRuntime({ omitViewportRoot: true, settledAtMs: 1_200 }); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + BROAD_TRANSITION_PARAMS, + ); + + assert.equal(outcome.settled, true); + assert.equal( + outcome.lastCapture?.snapshot.nodes.some((node) => node.label === 'action file'), + false, + ); + assert.ok( + outcome.waitedMs >= 1_500, + `settled partial transitional tree after ${outcome.waitedMs}ms`, + ); +}); + +test('settle keeps the default quiet window for a larger partial projection', async () => { + const { runtime } = transitionRuntime({ omitViewportRoot: true, settledAtMs: 1_200 }); + + const outcome = await runStableCaptureLoop( + runtime, + { session: 'default' }, + { + ...BROAD_TRANSITION_PARAMS, + broadTransitionBaselineNodes: elementSettingsSnapshot([1_138, 1_423, 2_144, 2_434]).nodes, + }, + ); + + assert.equal(outcome.settled, true); + assert.equal( + outcome.lastCapture?.snapshot.nodes.some((node) => node.label === 'action file'), + true, + ); + assert.ok(outcome.waitedMs < 800, `partial projection settled after ${outcome.waitedMs}ms`); +}); + test('settle honors an explicitly shorter quiet window across a broad replacement', async () => { const { runtime } = transitionRuntime(); @@ -149,14 +217,14 @@ test('settle honors an explicitly shorter quiet window across a broad replacemen assert.ok(outcome.waitedMs < 800, `short quiet window settled after ${outcome.waitedMs}ms`); }); -test('settle keeps the default quiet window for an overlapping local mutation', async () => { +test('settle keeps the default quiet window for a backend-less overlapping local mutation', async () => { const localMutation = { ...elementSettledRoomSnapshot, nodes: elementSettledRoomSnapshot.nodes.map((node) => node.label === 'Upload' ? { ...node, label: 'Add attachment' } : node, ), }; - const { runtime } = staticSnapshotRuntime(localMutation); + const { runtime } = staticSnapshotRuntime(withoutSnapshotBackend(localMutation, true)); const outcome = await runStableCaptureLoop( runtime, @@ -197,6 +265,8 @@ function transitionRuntime( options: { captureBackend?: 'tree' | 'private-ax'; firstCaptureKeepsBaseline?: boolean; + omitSnapshotBackend?: boolean; + omitViewportRoot?: boolean; settledAtMs?: number; sessionSnapshot?: SnapshotState; } = {}, @@ -217,13 +287,19 @@ function transitionRuntime( captureSnapshot: async () => { const keepsBaseline = options.firstCaptureKeepsBaseline === true && captures === 0; captures += 1; + const snapshot = keepsBaseline + ? elementThreadsNoticeSnapshot + : withCaptureBackend( + elapsedMs < settledAtMs ? elementTransientRoomSnapshot : elementSettledRoomSnapshot, + captureBackend, + ); return { - snapshot: keepsBaseline - ? elementThreadsNoticeSnapshot - : withCaptureBackend( - elapsedMs < settledAtMs ? elementTransientRoomSnapshot : elementSettledRoomSnapshot, - captureBackend, - ), + snapshot: withoutSnapshotBackend( + options.omitViewportRoot === true + ? { ...snapshot, nodes: snapshot.nodes.slice(1) } + : snapshot, + options.omitSnapshotBackend === true, + ), }; }, } satisfies AgentDeviceBackend, @@ -237,6 +313,12 @@ function transitionRuntime( return { runtime }; } +function withoutSnapshotBackend(snapshot: SnapshotState, omit: boolean): SnapshotState { + if (!omit) return snapshot; + const { backend: _backend, ...backendless } = snapshot; + return backendless; +} + function withCaptureBackend( snapshot: SnapshotState, backend: 'tree' | 'private-ax', diff --git a/src/commands/interaction/runtime/stable-capture.ts b/src/commands/interaction/runtime/stable-capture.ts index 1d0754686..9b9b5b23c 100644 --- a/src/commands/interaction/runtime/stable-capture.ts +++ b/src/commands/interaction/runtime/stable-capture.ts @@ -1,4 +1,5 @@ import type { SnapshotNode, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; +import { isViewportRootNode } from '@agent-device/contracts/snapshot'; import type { AgentDeviceRuntime, CommandContext } from '../../../runtime-contract.ts'; import { now, sleep } from '../../runtime-common.ts'; import { @@ -74,10 +75,7 @@ export async function runStableCaptureLoop( let deadlineMs = start + timeoutMs; let privateAxRecoveryBudgetReset = false; const session = await runtime.sessions.get(options.session ?? 'default'); - const transitionBaseline = stableCaptureTransitionBaseline( - params.broadTransitionBaselineNodes, - session?.snapshot, - ); + let transitionBaseline: StableCaptureSignal | undefined; let preferredBackend = preferredSnapshotBackendForVerdict(session?.snapshot?.snapshotQuality); // Cadence derives from the quiet window (never slower than the default // poll): a caller asking for a 50ms quiet window should not be forced onto a @@ -108,6 +106,11 @@ export async function runStableCaptureLoop( } captures += 1; lastCapture = capture; + transitionBaseline ??= stableCaptureTransitionBaseline( + params.broadTransitionBaselineNodes, + capture.snapshot, + runtime.backend.platform, + ); const signal = stableCaptureSignal(capture.snapshot); requiredQuietMs = stableCaptureTransitionQuietMs({ baseline: transitionBaseline, @@ -179,10 +182,34 @@ export async function runStableCaptureLoop( function stableCaptureTransitionBaseline( baselineNodes: SnapshotNode[] | undefined, snapshot: Parameters[0] | undefined, + platform: AgentDeviceRuntime['backend']['platform'], ): StableCaptureSignal | undefined { - return baselineNodes && snapshot?.backend === 'xctest' - ? stableCaptureSignal({ ...snapshot, nodes: baselineNodes }) - : undefined; + // SnapshotState.backend is optional at the runtime boundary. The bound iOS backend remains + // authoritative when a presented capture omits that provenance; a declared non-XCTest backend + // still wins so this confirmation cannot leak onto another capture implementation. + if (!baselineNodes || !snapshot || !isBoundIosXCTestCapture(snapshot.backend, platform)) { + return undefined; + } + // A broad *screen* replacement needs a complete post-action viewport projection. The baseline + // comes from settle's authoritative pre-action capture, but a presented iOS modal can omit its + // Application root and still be the complete interaction surface. Requiring the root only from + // the candidate prevents scoped/synthetic post-action fragments from extending settle latency. + const hasCompleteViewportProjection = snapshot.nodes.some(isViewportRootNode); + // Tiny pre-action surfaces are the other trustworthy completeness signal: iOS presents alerts + // and sheets as a handful of nodes, and their first post-dismissal capture can temporarily omit + // the viewport root. Treating that rootless replacement as an arbitrary scoped projection lets + // a coherent transitional tree win the default 500ms quiet window. + const hasTinyPresentedBaseline = baselineNodes.length <= TINY_STABLE_TREE_NODE_COUNT; + if (!hasCompleteViewportProjection && !hasTinyPresentedBaseline) return undefined; + return stableCaptureSignal({ ...snapshot, nodes: baselineNodes }); +} + +function isBoundIosXCTestCapture( + backend: Parameters[0]['backend'], + platform: AgentDeviceRuntime['backend']['platform'], +): boolean { + if (backend === 'xctest') return true; + return backend === undefined && platform === 'ios'; } function stableCaptureTransitionQuietMs(params: {