diff --git a/packages/client/app/components/UsageStatusModal.vue b/packages/client/app/components/UsageStatusModal.vue index 630a72d5..1be1983d 100644 --- a/packages/client/app/components/UsageStatusModal.vue +++ b/packages/client/app/components/UsageStatusModal.vue @@ -4,6 +4,7 @@ import { useProjects } from '../composables/useProjects' import { useRpc } from '../composables/useRpc' import { normalizeAccountRateLimits, + mergeAccountRateLimits, formatRateLimitWindowDuration, type RateLimitBucket } from '../../shared/account-rate-limits' @@ -99,8 +100,10 @@ const closeUsageStatus = () => { resetUsageStatusState() } -const applyUsageStatusSnapshot = (value: unknown) => { - buckets.value = normalizeAccountRateLimits(value) +const applyUsageStatusSnapshot = (value: unknown, sparse = false) => { + buckets.value = sparse + ? mergeAccountRateLimits(buckets.value, value) + : normalizeAccountRateLimits(value) error.value = null } @@ -130,7 +133,7 @@ const openUsageStatus = async () => { return } - applyUsageStatusSnapshot(notification.params) + applyUsageStatusSnapshot(notification.params, true) loading.value = false }) diff --git a/packages/client/shared/account-rate-limits.ts b/packages/client/shared/account-rate-limits.ts index 5f97f9d8..6367f705 100644 --- a/packages/client/shared/account-rate-limits.ts +++ b/packages/client/shared/account-rate-limits.ts @@ -124,7 +124,11 @@ const normalizeRateLimitBucket = (value: unknown): RateLimitBucket | null => { const collectRateLimitCandidates = (value: unknown) => { const root = isObjectRecord(value) ? value : null - const rateLimits = Array.isArray(root?.rateLimits) ? root.rateLimits : [] + const rateLimits = Array.isArray(root?.rateLimits) + ? root.rateLimits + : isObjectRecord(root?.rateLimits) + ? [root.rateLimits] + : [] const rateLimitsByLimitId = isObjectRecord(root?.rateLimitsByLimitId) ? Object.values(root.rateLimitsByLimitId) : [] @@ -133,7 +137,13 @@ const collectRateLimitCandidates = (value: unknown) => { return [...rateLimits, ...rateLimitsByLimitId] } - return Array.isArray(value) ? value : [] + if (Array.isArray(value)) { + return value + } + + return root && ('primary' in root || 'secondary' in root) + ? [root] + : [] } export const normalizeAccountRateLimits = (value: unknown): RateLimitBucket[] => { @@ -166,6 +176,18 @@ export const normalizeAccountRateLimits = (value: unknown): RateLimitBucket[] => }) } +/** + * Merge a sparse rolling update into the last authoritative snapshot. + * Missing/null values in an update never erase facts already observed. + */ +export const mergeAccountRateLimits = ( + existing: readonly RateLimitBucket[], + update: unknown +): RateLimitBucket[] => normalizeAccountRateLimits([ + ...existing, + ...normalizeAccountRateLimits(update) +]) + export const formatRateLimitWindowDuration = (value: number | null) => { if (value == null || value <= 0) { return null diff --git a/packages/webxr/README.md b/packages/webxr/README.md index 8f71a61a..85d79a35 100644 --- a/packages/webxr/README.md +++ b/packages/webxr/README.md @@ -7,11 +7,11 @@ The package is a progressive enhancement. The existing dashboard remains the pri ## Requirements - A secure context. `https://` is required on a remote headset; the browser's `localhost` exception does not apply to a plain LAN IP. -- A browser reporting `navigator.xr.isSessionSupported('immersive-vr')`. +- A browser reporting `immersive-vr` or `immersive-ar` support. VR is preferred when both are available; AR-only devices enter their supported mode. - A materialized Codori project or projectless-chat thread. - The Codori server started with realtime voice enabled if voice controls are required. -The entry screen checks support without user-agent sniffing. `requestSession('immersive-vr')` and microphone access occur only after explicit user actions. +The entry screen probes the two modes independently without user-agent sniffing, so a failed AR probe does not disable working VR. `requestSession()` and microphone access occur only after explicit user actions. ## Development @@ -32,6 +32,11 @@ file-change, tool, search, and background-terminal surfaces without connecting to a live workspace. This fixture is useful for browser-side text density, border, color, and layout comparisons; it does not reproduce headset framebuffer scaling or fixed foveation. +The kitchen-sink fixture also opens the lime status window with primary and +secondary quota windows, context state, and the initial action registry. +Append `&blend=alpha-blend` or `&blend=additive` to preview the two +development-only agent contrast treatments. These are visual fixtures, not +claims that a camera or optical display is active. Vite proxies `/api` HTTP and WebSocket traffic to `http://127.0.0.1:4310` by default. Point it at another running Codori server @@ -56,6 +61,19 @@ Users sensitive to motion or flicker should enable reduced effects before enteri ## Controls +Status window: + +- The verified `menu` component on the left `htc-vive-focus` input profile toggles the window. Codori does not guess extra gamepad indices on other profiles, and WebXR-reserved app/system buttons may be absent. +- With a tracked left hand and no active left controller, hold the raised back-of-hand pose for `450 ms`. Lowering it for `180 ms` dismisses the window; short pose/tracking gaps use hysteresis and a `300 ms` tracking-loss grace. +- A controller-opened window remains open until reinvoked, an action is selected, or the left grip is held below the lowering threshold for `250 ms`. +- Placement uses the left grip or WebXR `wrist` joint plus a viewer-facing vertical offset. WebXR exposes no elbow joint, so this is a wrist/grip-and-view approximation rather than full forearm tracking. +- Right controller rays and direct controller contact can activate actions. Tracked hands require direct `index-finger-tip` contact; remote pinch/ray activation is rejected for status actions. +- When no exposed mapped menu component or gesture-eligible tracked left hand can invoke status, an in-canvas bottom-right `Menu` is shown. This includes right-hand-only tracking and controllers whose app/system button is reserved or unmapped; their target ray can select the fallback. A left controller takes precedence over the left-hand gesture, so an unmapped left controller keeps the fallback available for ray selection. A DOM-overlay button mirrors it when the optional DOM Overlays feature is granted. Both disappear when a mapped menu control or eligible left-hand gesture becomes usable; screen/gaze can open the menu but cannot bypass `controller-or-touch` action policy. + +The status view uses a translucent pale lime treatment distinct from cyan panes. It shows the authoritative primary and secondary Codex quota windows, localized reset times, current-thread context remaining only when known, connection/voice state, pane count, and thread/workspace identity. Sparse `account/rateLimits/updated` buckets merge by `limitId` into the last `account/rateLimits/read` snapshot; unknown quota or context is labeled unavailable rather than rendered as zero/full. + +Initial actions are `Passthrough`, `Recenter workspace`, the live voice/resume-audio action, `Reduced effects`, and `Exit immersive`. Each registry item carries a stable id, state, availability/disabled reason, callback, and input policy so additional actions can be added without changing the status surface architecture. + Controller: - target ray: hover and select controls or panel content @@ -75,11 +93,15 @@ Tracked hand: Native and synthesized primary actions are de-duplicated. Competing grabs have one deterministic owner, and input-source loss releases hover/grab state. +`Recenter workspace` rotates and translates one shared anchor into the current horizontal gaze at clamped eye height. The agent light, transcript, status surfaces, automatic panes, and manually moved pane-local transforms move together without reallocating pane ids. A `local-floor` reference-space `reset` schedules exactly one anchor refresh; Codori does not emulate or require a reserved platform recenter button. + +`Passthrough` is a real session-mode transition, not a background toggle. When `immersive-ar` is supported, Codori ends the current session and requests the other mode while retaining the same workspace and voice runtimes/subscriptions. If the browser cannot complete that transition within the action, the entry surface explains that state and offers explicit re-entry without recreating the RPC runtime. In a non-opaque AR session, transparent renderer pixels expose the environment and room geometry plus the boundary Exit door are hidden. `alpha-blend` uses a restrained dark stipple shell outside the agent light; `additive` uses a bright magenta shape outline because dark pixels cannot occlude an optical display. An AR session reporting `opaque` is never described as passthrough. The normal projection path remains correct without WebXR Layers. + Immersive entry starts the realtime voice session automatically after at least `500 ms` and as soon as the workspace runtime is ready. New XR sessions reuse the voice selection and browser-only voice-instruction override saved by Settings → Voice. The selected voice is sent only when the connected server advertises it; prompt precedence remains browser override, `experimental_realtime_ws_backend_prompt` from `config.toml`, then the Codori default. Selecting the central light stops the active session and re-arms the dormant visual state; selecting it again replays the full awakening before restarting voice. A door-sized rounded `Exit` surface sits on the `10 m × 10 m` room boundary, beyond the agent light along the initial view direction. The 2D fallback remains available before immersive entry. While realtime startup is pending, the agent light remains dormant at `86%` scale, `72%` intensity, and nearly zero lens flare. Startup triggers a `160 ms` flare ignition followed by an `850 ms` settle into the current activity state. Reduced-effects mode uses a smaller scale excursion and lower flare peak. -Web Audio synthesizes a one-second agent-awakening cue whose low mechanical chord clusters and softer upper harmonics beat against each other while their pitch rises on an ease-out curve for `700 ms`, followed by a `300 ms` fade whose cubic curve preserves the initial resonance before dropping away. Panel appearance uses a separate `250 ms` cue that blends an immediate low body with delayed, beating high harmonics. Multiple panels created in one synchronization batch produce one softly amplified cue instead of overlapping sounds. Run `pnpm --filter @codori/webxr render:sfx-previews -- ` to render listenable WAV previews from the same canonical sound plans. +Web Audio synthesizes a one-second agent-awakening cue whose low mechanical chord clusters and softer upper harmonics beat against each other while their pitch rises on an ease-out curve for `700 ms`, followed by a `300 ms` fade whose cubic curve preserves the initial resonance before dropping away. Panel appearance uses a separate `250 ms` cue that blends an immediate low body with delayed, beating high harmonics. The status window uses related `380 ms` pitch-up and `300 ms` pitch-down cues. Multiple panels created in one synchronization batch produce one softly amplified cue instead of overlapping sounds. Run `pnpm --filter @codori/webxr render:sfx-previews -- ` to render listenable WAV previews from the same canonical sound plans. ## Panel semantics and caps @@ -102,4 +124,4 @@ Growing output follows the live tail until manual scrolling. Later deltas preser Automated tests cover session options/failure states, deterministic light bounds and reduced effects, transcript generation plus 30-second visibility and 250 ms scale transitions, streaming panel lifecycle plus 60-second dwell and 125 ms forced dismissal, panel retention/layout/scroll state, input ownership and source loss, shared notification adapters, `/xr/` server routing, and package builds. -Real headset validation is still required before making device-specific support or performance claims. Record the headset OS, browser version, optional features granted, target refresh rate, median frame time, sustained worst frame-time band, text legibility, and a 15-minute mixed voice/tool memory observation for each supported device/browser combination. +Real headset validation is still required before making device-specific support, transition, input-component, blend-mode, or performance claims. For each controller, hand-tracking, and screen/gaze device/browser combination, record headset/device, OS, browser version, supported session modes, actual `environmentBlendMode` and `interactionMode`, optional features granted, exposed input profiles/components, app-visible buttons, target refresh rate, median frame time, sustained worst frame-time band, status anchoring/direct-touch behavior, alpha/additive contrast readability, text legibility, and a 15-minute mixed voice/tool memory observation. Browser kitchen-sink QA proves only canvas layout, colors, typography, and animation; it cannot prove headset poses, camera passthrough, additive optics, reserved buttons, or seamless session switching. diff --git a/packages/webxr/index.html b/packages/webxr/index.html index e1940d79..8d17054b 100644 --- a/packages/webxr/index.html +++ b/packages/webxr/index.html @@ -27,6 +27,7 @@

Step into your coding session.

diff --git a/packages/webxr/scripts/render-sound-previews.mjs b/packages/webxr/scripts/render-sound-previews.mjs index 8cc56b42..70e9ed4b 100644 --- a/packages/webxr/scripts/render-sound-previews.mjs +++ b/packages/webxr/scripts/render-sound-previews.mjs @@ -163,7 +163,9 @@ await mkdir(outputDirectory, { recursive: true }) for (const [name, plan] of [ ['agent-awakening.wav', soundEffectPlans.awakening], - ['panel-appear.wav', soundEffectPlans.panelAppear] + ['panel-appear.wav', soundEffectPlans.panelAppear], + ['status-open.wav', soundEffectPlans.statusOpen], + ['status-close.wav', soundEffectPlans.statusClose] ]) { const outputPath = path.join(outputDirectory, name) await writeFile(outputPath, wavBuffer(renderPlan(plan))) diff --git a/packages/webxr/src/billboard.ts b/packages/webxr/src/billboard.ts index 6ac94ed0..46d5d821 100644 --- a/packages/webxr/src/billboard.ts +++ b/packages/webxr/src/billboard.ts @@ -1,8 +1,16 @@ -import { Matrix4, Quaternion, Vector3 } from 'three' +import { + Matrix4, + Quaternion, + Vector3, + type Object3D +} from 'three' const lookAtMatrix = new Matrix4() const targetQuaternion = new Quaternion() const worldUp = new Vector3(0, 1, 0) +const billboardWorldPosition = new Vector3() +const billboardWorldQuaternion = new Quaternion() +const billboardParentQuaternion = new Quaternion() export const viewerFacingQuaternion = ( objectPosition: Vector3, @@ -26,3 +34,22 @@ export const smoothViewerFacingQuaternion = ( const progress = 1 - Math.exp(-Math.max(0, damping) * Math.max(0, deltaSeconds)) return current.slerp(target, progress) } + +export const viewerFacingLocalQuaternion = ( + object: Object3D, + viewerPosition: Vector3, + target = new Quaternion() +) => { + object.getWorldPosition(billboardWorldPosition) + billboardWorldQuaternion.copy(viewerFacingQuaternion( + billboardWorldPosition, + viewerPosition + )) + if (!object.parent) { + return target.copy(billboardWorldQuaternion) + } + object.parent.getWorldQuaternion(billboardParentQuaternion) + return target.copy(billboardParentQuaternion) + .invert() + .multiply(billboardWorldQuaternion) +} diff --git a/packages/webxr/src/immersive-scene.ts b/packages/webxr/src/immersive-scene.ts index 88d914b4..8947c240 100644 --- a/packages/webxr/src/immersive-scene.ts +++ b/packages/webxr/src/immersive-scene.ts @@ -1,6 +1,8 @@ import { AmbientLight, + BufferGeometry, Color, + Float32BufferAttribute, GridHelper, Group, LineSegments, @@ -8,6 +10,9 @@ import { MeshBasicMaterial, PerspectiveCamera, PlaneGeometry, + Points, + PointsMaterial, + TorusGeometry, Scene, SRGBColorSpace, Timer, @@ -17,6 +22,7 @@ import { import { AgentLightView } from './agent-light-view' import { viewerFacingQuaternion, + viewerFacingLocalQuaternion, smoothViewerFacingQuaternion } from './billboard' import { @@ -41,6 +47,19 @@ import { import { TranscriptBubbleView } from './transcript-bubble-view' import { WorldControls, type WorldControlAction } from './world-controls' import { WorldStatus } from './world-status' +import { + StatusWindowView +} from './status-window-view' +import type { + StatusActionId, + StatusWindowInvocation, + StatusWindowSnapshot +} from './status-window-model' +import type { ImmersiveSessionMode } from './xr-capability' +import { + ReferenceSpaceResetModel, + resolveWorkspaceAnchor +} from './workspace-anchor' export type ImmersiveSceneOptions = { canvas: HTMLCanvasElement @@ -52,6 +71,10 @@ export type ImmersiveSceneOptions = { onPanelFocused: (panelId: string, position: Vector3) => void onPanelDismiss: (panelId: string) => void onPanelAppeared: (panelCount: number) => void + onStatusAction: (action: StatusActionId) => void + onStatusOpened: () => void + onStatusClosed: () => void + onStatusFallbackChanged: (visible: boolean) => void } const viewerPosition = new Vector3() @@ -59,9 +82,10 @@ const viewerDirection = new Vector3() const worldCenter = new Vector3(0, 1.65, 0) const worldForward = new Vector3(0, 0, -1) const floorCenter = new Vector3() - -const clamp = (value: number, minimum: number, maximum: number) => - Math.min(maximum, Math.max(minimum, value)) +const statusAnchorPosition = new Vector3() +const menuWorldPosition = new Vector3() +const menuOffset = new Vector3(0.52, -0.32, -1.15) +const fallbackStatusOffset = new Vector3(-0.18, 0, -1.12) export class ImmersiveScene { readonly scene = new Scene() @@ -88,6 +112,10 @@ export class ImmersiveScene { private readonly status = new WorldStatus() + private readonly statusWindow = new StatusWindowView() + + private readonly contrast = new Group() + private readonly panels = new Map() private readonly interaction: ImmersiveInteractionSystem @@ -106,11 +134,21 @@ export class ImmersiveScene { private disposed = false + private readonly referenceReset = new ReferenceSpaceResetModel() + + private releaseReferenceReset: (() => void) | null = null + + private sessionMode: ImmersiveSessionMode = 'immersive-vr' + + private environmentBlendMode: XREnvironmentBlendMode = 'opaque' + + private statusInvocation: StatusWindowInvocation | null = null + constructor(private readonly options: ImmersiveSceneOptions) { this.renderer = new WebGLRenderer({ canvas: options.canvas, antialias: true, - alpha: false, + alpha: true, powerPreference: 'high-performance' }) this.renderer.outputColorSpace = SRGBColorSpace @@ -128,8 +166,11 @@ export class ImmersiveScene { this.agentLight.group, this.transcriptView.group, this.controls.group, - this.status.group + this.status.group, + this.contrast ) + this.scene.add(this.statusWindow.group, this.statusWindow.menuGroup) + this.createContrastTreatments() this.setWorldCenter(new Vector3(0, 1.65, 0)) this.interaction = new ImmersiveInteractionSystem({ @@ -140,12 +181,26 @@ export class ImmersiveScene { this.agentLight.hitTarget, ...this.controls.hitTargets ], + getStatusTargets: () => this.statusWindow.actionHits, + getStatusMenuTarget: () => this.statusWindow.menuHit, + isStatusOpen: () => this.statusWindow.isOpen, + getStatusInvocation: () => this.statusInvocation, onScroll: options.onPanelScroll, onPanelInteracted: options.onPanelInteracted, onPanelMoved: options.onPanelMoved, onPanelFocused: options.onPanelFocused, onPanelDismiss: options.onPanelDismiss, - onAction: options.onAction + onAction: options.onAction, + onStatusToggle: invocation => this.toggleStatusWindow(invocation), + onStatusDismiss: () => this.closeStatusWindow(), + onStatusAction: (action) => { + this.closeStatusWindow() + options.onStatusAction(action) + }, + onInputCapabilitiesChanged: ({ fallbackMenu }) => { + this.statusWindow.setMenuVisible(fallbackMenu) + options.onStatusFallbackChanged(fallbackMenu) + } }) this.renderer.setAnimationLoop((timestamp) => { this.renderFrame(timestamp) @@ -153,6 +208,53 @@ export class ImmersiveScene { this.resize() } + private createContrastTreatments() { + const ditherPositions: number[] = [] + const pointCount = 420 + const goldenAngle = Math.PI * (3 - Math.sqrt(5)) + for (let index = 0; index < pointCount; index += 1) { + const y = 1 - ((index / (pointCount - 1)) * 2) + const radius = Math.sqrt(1 - y * y) + const theta = goldenAngle * index + ditherPositions.push( + Math.cos(theta) * radius * 0.34, + y * 0.34, + Math.sin(theta) * radius * 0.34 + ) + } + const ditherGeometry = new BufferGeometry() + ditherGeometry.setAttribute( + 'position', + new Float32BufferAttribute(ditherPositions, 3) + ) + const dither = new Points( + ditherGeometry, + new PointsMaterial({ + color: '#071008', + size: 0.012, + sizeAttenuation: true, + transparent: true, + opacity: 0.38, + depthWrite: false + }) + ) + dither.name = 'alpha-passthrough-dither-shell' + dither.userData.contrast = 'dither' + const additive = new Mesh( + new TorusGeometry(0.28, 0.025, 10, 48), + new MeshBasicMaterial({ + color: '#ff6bd6', + transparent: true, + opacity: 0.88, + depthWrite: false + }) + ) + additive.name = 'additive-passthrough-shape-outline' + additive.userData.contrast = 'additive-shape' + this.contrast.add(dither, additive) + this.contrast.visible = false + } + private createRoom() { this.world.add(this.room) const floorMaterial = new MeshBasicMaterial({ @@ -213,20 +315,27 @@ export class ImmersiveScene { } else { worldForward.normalize() } - this.agentLight.group.position.copy(center) + this.world.position.set(center.x, 0, center.z) + this.world.quaternion.setFromUnitVectors( + new Vector3(0, 0, -1), + worldForward + ) + worldCenter.set(0, center.y, 0) + this.agentLight.group.position.copy(worldCenter) this.transcriptView.group.position.set( - center.x, + 0, center.y + 0.72, - center.z + 0 ) - this.controls.placeExitDoor(center, worldForward) + this.controls.placeExitDoor(worldCenter, new Vector3(0, 0, -1)) this.status.group.position.set( - center.x, + 0, center.y + 1.22, - center.z + 0 ) - floorCenter.set(center.x, 0, center.z) + floorCenter.set(0, 0, 0) this.room.position.copy(floorCenter) + this.contrast.position.copy(this.agentLight.group.position) } private placeFromInitialViewer() { @@ -242,14 +351,14 @@ export class ImmersiveScene { } else { viewerDirection.normalize() } - const center = viewerPosition.clone() - .addScaledVector(viewerDirection, INITIAL_LIGHT_DISTANCE_METERS) - center.y = clamp( - viewerPosition.y, - MIN_LIGHT_HEIGHT_METERS, - MAX_LIGHT_HEIGHT_METERS - ) - this.setWorldCenter(center, viewerDirection) + const anchor = resolveWorkspaceAnchor({ + viewerPosition, + viewerDirection, + distanceMeters: INITIAL_LIGHT_DISTANCE_METERS, + minimumHeightMeters: MIN_LIGHT_HEIGHT_METERS, + maximumHeightMeters: MAX_LIGHT_HEIGHT_METERS + }) + this.setWorldCenter(anchor.position, anchor.forward) this.placedFromViewer = true this.syncPanelViews() } @@ -286,6 +395,84 @@ export class ImmersiveScene { } } + setStatusWindowSnapshot(snapshot: StatusWindowSnapshot) { + this.statusWindow.setSnapshot(snapshot) + } + + private toggleStatusWindow(invocation: StatusWindowInvocation) { + if (this.statusWindow.toggle(performance.now())) { + if (this.statusWindow.isOpen) { + this.statusInvocation = invocation + this.options.onStatusOpened() + } else { + this.statusInvocation = null + this.options.onStatusClosed() + } + } + } + + toggleStatusFromFallback() { + this.toggleStatusWindow('fallback') + } + + openStatusForPreview() { + if (!this.statusWindow.isOpen) { + this.toggleStatusWindow('fallback') + } + } + + private closeStatusWindow() { + if (this.statusWindow.close(performance.now())) { + this.statusInvocation = null + this.options.onStatusClosed() + } + } + + recenterWorkspace() { + const camera = this.renderer.xr.isPresenting + ? this.renderer.xr.getCamera() + : this.camera + camera.getWorldPosition(viewerPosition) + camera.getWorldDirection(viewerDirection) + viewerDirection.y = 0 + if (viewerDirection.lengthSq() < 0.001) { + viewerDirection.set(0, 0, -1) + } else { + viewerDirection.normalize() + } + const anchor = resolveWorkspaceAnchor({ + viewerPosition, + viewerDirection, + distanceMeters: INITIAL_LIGHT_DISTANCE_METERS, + minimumHeightMeters: MIN_LIGHT_HEIGHT_METERS, + maximumHeightMeters: MAX_LIGHT_HEIGHT_METERS + }) + this.setWorldCenter(anchor.position, anchor.forward) + this.syncPanelViews() + } + + setSessionVisualMode( + mode: ImmersiveSessionMode, + environmentBlendMode: XREnvironmentBlendMode + ) { + this.sessionMode = mode + this.environmentBlendMode = environmentBlendMode + const passthrough = mode === 'immersive-ar' + && environmentBlendMode !== 'opaque' + this.room.visible = !passthrough + this.controls.group.visible = !passthrough + this.scene.background = passthrough ? null : new Color('#01040a') + this.renderer.setClearColor(0x000000, passthrough ? 0 : 1) + this.contrast.visible = passthrough + for (const child of this.contrast.children) { + child.visible = environmentBlendMode === 'alpha-blend' + ? child.userData.contrast === 'dither' + : environmentBlendMode === 'additive' + ? child.userData.contrast === 'additive-shape' + : false + } + } + private syncPanelViews() { const layoutNow = performance.now() let appearedPanelCount = 0 @@ -299,9 +486,9 @@ export class ImmersiveScene { } const placements = allocatePanelSlots(this.panelSnapshots, { - x: worldCenter.x, + x: 0, y: 0, - z: worldCenter.z + z: 0 }) const placementById = new Map( placements.map(placement => [placement.id, placement]) @@ -344,7 +531,12 @@ export class ImmersiveScene { this.renderer.setSize(width, height, false) } - async setSession(session: XRSession | null) { + async setSession( + session: XRSession | null, + mode: ImmersiveSessionMode = 'immersive-vr' + ) { + this.releaseReferenceReset?.() + this.releaseReferenceReset = null if (session) { this.placedFromViewer = false this.lightAnimator.enterDormant() @@ -352,6 +544,21 @@ export class ImmersiveScene { this.lightAnimator.resetAwakening() } await this.renderer.xr.setSession(session) + if (session) { + this.setSessionVisualMode(mode, session.environmentBlendMode) + const referenceSpace = this.renderer.xr.getReferenceSpace() + if (referenceSpace) { + const handleReset = () => { + this.referenceReset.mark() + } + referenceSpace.addEventListener('reset', handleReset) + this.releaseReferenceReset = () => { + referenceSpace.removeEventListener('reset', handleReset) + } + } + } else { + this.setSessionVisualMode('immersive-vr', 'opaque') + } } private renderFrame(timestamp: number) { @@ -371,12 +578,52 @@ export class ImmersiveScene { timeSeconds ) this.placeFromInitialViewer() + if (this.referenceReset.take()) { + this.recenterWorkspace() + } const camera = this.renderer.xr.isPresenting ? this.renderer.xr.getCamera() : this.camera camera.getWorldPosition(viewerPosition) const now = performance.now() + const anchor = this.interaction.statusAnchor() + if (anchor && this.statusWindow.group.visible) { + anchor.getWorldPosition(statusAnchorPosition) + this.statusWindow.group.position.copy(statusAnchorPosition) + this.statusWindow.group.position.y += 0.34 + const statusTarget = viewerFacingQuaternion( + this.statusWindow.group.position, + viewerPosition + ) + smoothViewerFacingQuaternion( + this.statusWindow.group.quaternion, + statusTarget, + deltaSeconds + ) + } else if (this.statusWindow.group.visible) { + this.statusWindow.group.position.copy(fallbackStatusOffset) + .applyQuaternion(camera.quaternion) + .add(viewerPosition) + const fallbackTarget = viewerFacingQuaternion( + this.statusWindow.group.position, + viewerPosition + ) + smoothViewerFacingQuaternion( + this.statusWindow.group.quaternion, + fallbackTarget, + deltaSeconds + ) + } + menuWorldPosition.copy(menuOffset).applyQuaternion(camera.quaternion) + .add(viewerPosition) + this.statusWindow.menuGroup.position.copy(menuWorldPosition) + const menuTarget = viewerFacingQuaternion( + this.statusWindow.menuGroup.position, + viewerPosition + ) + this.statusWindow.menuGroup.quaternion.copy(menuTarget) + this.statusWindow.update(now, this.options.reducedEffects()) this.transcriptView.update( this.transcriptModel.update( this.transcriptSegments, @@ -385,43 +632,27 @@ export class ImmersiveScene { ), now ) - const bubbleTarget = viewerFacingQuaternion( - this.transcriptView.group.position, - viewerPosition - ) smoothViewerFacingQuaternion( this.transcriptView.group.quaternion, - bubbleTarget, + viewerFacingLocalQuaternion(this.transcriptView.group, viewerPosition), deltaSeconds ) - const controlsTarget = viewerFacingQuaternion( - this.controls.group.position, - viewerPosition - ) smoothViewerFacingQuaternion( this.controls.group.quaternion, - controlsTarget, + viewerFacingLocalQuaternion(this.controls.group, viewerPosition), deltaSeconds ) - const statusTarget = viewerFacingQuaternion( - this.status.group.position, - viewerPosition - ) smoothViewerFacingQuaternion( this.status.group.quaternion, - statusTarget, + viewerFacingLocalQuaternion(this.status.group, viewerPosition), deltaSeconds ) for (const view of this.panels.values()) { view.updateAnimation(now) - const target = viewerFacingQuaternion( - view.group.position, - viewerPosition - ) smoothViewerFacingQuaternion( view.group.quaternion, - target, + viewerFacingLocalQuaternion(view.group, viewerPosition), deltaSeconds ) } @@ -437,6 +668,8 @@ export class ImmersiveScene { this.renderer.setAnimationLoop(null) this.timer.dispose() this.interaction.dispose() + this.releaseReferenceReset?.() + this.releaseReferenceReset = null for (const view of this.panels.values()) { view.dispose() } @@ -444,9 +677,14 @@ export class ImmersiveScene { this.transcriptView.dispose() this.controls.dispose() this.status.dispose() + this.statusWindow.dispose() this.agentLight.dispose() this.scene.traverse((object) => { - if (object instanceof Mesh || object instanceof LineSegments) { + if ( + object instanceof Mesh + || object instanceof LineSegments + || object instanceof Points + ) { object.geometry.dispose() if (Array.isArray(object.material)) { object.material.forEach(material => material.dispose()) diff --git a/packages/webxr/src/interaction-system.ts b/packages/webxr/src/interaction-system.ts index 2a7a661a..83592e86 100644 --- a/packages/webxr/src/interaction-system.ts +++ b/packages/webxr/src/interaction-system.ts @@ -1,4 +1,5 @@ import { + Box3, BufferGeometry, Line, LineBasicMaterial, @@ -24,6 +25,17 @@ import { } from './panel-interaction' import type { SpatialPanelView } from './panel-view' import type { WorldControlAction } from './world-controls' +import { + canActivateStatusAction, + mappedMenuButtonIndex, + shouldShowStatusFallbackMenu, + StatusControllerArmModel, + StatusGestureModel, + type StatusActionId, + type StatusActionInputPolicy, + type StatusWindowInvocation, + type StatusActivation +} from './status-window-model' type SourceRuntime = { id: string @@ -41,6 +53,8 @@ type SourceRuntime = { grabSphere: Sphere grabInitialPosition: Vector3 grabMoved: boolean + menuPressed: boolean + contactActionId: StatusActionId | null listeners: { connected: (event: unknown) => void disconnected: () => void @@ -56,12 +70,24 @@ export type InteractionSystemOptions = { root: Object3D getPanels: () => ReadonlyMap getControlTargets: () => readonly Mesh[] + getStatusTargets: () => readonly Mesh[] + getStatusMenuTarget: () => Mesh | null + isStatusOpen: () => boolean + getStatusInvocation: () => StatusWindowInvocation | null onScroll: (panelId: string, deltaLines: number) => void onPanelInteracted: (panelId: string) => void onPanelMoved: (panelId: string, position: Vector3) => void onPanelFocused: (panelId: string, position: Vector3) => void onPanelDismiss: (panelId: string) => void onAction: (action: WorldControlAction) => void + onStatusToggle: (invocation: StatusWindowInvocation) => void + onStatusDismiss: () => void + onStatusAction: (action: StatusActionId) => void + onInputCapabilitiesChanged: (input: { + controller: boolean + hand: boolean + fallbackMenu: boolean + }) => void } const rayOrigin = new Vector3() @@ -76,6 +102,10 @@ const panelPlane = new Plane() const panelPlaneNormal = new Vector3() const panelPlanePosition = new Vector3() const panelPlaneQuaternion = new Quaternion() +const jointQuaternion = new Quaternion() +const handBackNormal = new Vector3() +const wristToViewer = new Vector3() +const contactBounds = new Box3() const PANEL_GRAB_TAP_MAX_DISTANCE_METERS = 0.12 const PANEL_FOCUSED_DISTANCE_METERS = 1.8 @@ -95,6 +125,48 @@ export const resolveRayGrabPosition = ( return intersection?.add(offset) ?? null } +export const resolveTrackedHandJoint = ( + hand: XRHandSpace, + name: XRHandJoint +) => { + const joint = hand.joints[name] + return hand.visible && joint?.visible ? joint : null +} + +export const mappedStatusMenuButtonIndex = ( + source: Pick +) => { + const index = mappedMenuButtonIndex(source.handedness, source.profiles) + return index != null && source.gamepad?.buttons[index] + ? index + : null +} + +export const resolveStatusFallbackMenuVisibility = ( + sources: readonly Pick[] +) => { + const trackedLeftHand = sources.some(runtime => + runtime.inputSource?.handedness === 'left' + && Boolean(runtime.inputSource.hand) + && resolveTrackedHandJoint(runtime.hand, 'wrist') !== null + ) + const leftControllerActive = sources.some(runtime => + runtime.inputSource?.handedness === 'left' + && !runtime.inputSource.hand + && runtime.inputSource.targetRayMode === 'tracked-pointer' + ) + const mappedMenuController = sources.some(runtime => + runtime.inputSource != null + && !runtime.inputSource.hand + && mappedStatusMenuButtonIndex(runtime.inputSource) !== null + ) + return shouldShowStatusFallbackMenu({ + mappedMenuController, + trackedLeftHand, + leftControllerActive + }) +} + export const resolveFocusedPanelPosition = ( viewer: Vector3, panel: Vector3, @@ -114,6 +186,30 @@ export const resolveFocusedPanelPosition = ( ) } +export const worldPointToPanelLocal = ( + panel: Object3D, + worldPoint: Vector3, + target = new Vector3() +) => { + target.copy(worldPoint) + panel.parent?.worldToLocal(target) + return target +} + +export const resolveFocusedPanelLocalPosition = ( + viewerWorld: Vector3, + panel: Object3D, + target = new Vector3() +) => { + panel.getWorldPosition(panelPlanePosition) + resolveFocusedPanelPosition( + viewerWorld, + panelPlanePosition, + target + ) + return worldPointToPanelLocal(panel, target, target) +} + export const resolveRayPanelPosition = ( ray: Ray, panel: Object3D, @@ -139,6 +235,12 @@ export class ImmersiveInteractionSystem { private readonly sources: SourceRuntime[] = [] + private readonly statusGesture = new StatusGestureModel() + + private readonly statusControllerArm = new StatusControllerArmModel() + + private lastInputCapabilities = '' + private disposed = false constructor(private readonly options: InteractionSystemOptions) { @@ -196,6 +298,8 @@ export class ImmersiveInteractionSystem { grabSphere: new Sphere(), grabInitialPosition: new Vector3(), grabMoved: false, + menuPressed: false, + contactActionId: null, listeners: null } const listeners = { @@ -267,6 +371,14 @@ export class ImmersiveInteractionSystem { } } targets.push(...this.options.getControlTargets()) + if (this.options.isStatusOpen()) { + targets.push(...this.options.getStatusTargets()) + } else { + const menu = this.options.getStatusMenuTarget() + if (menu?.visible && menu.parent?.visible) { + targets.push(menu) + } + } return targets } @@ -307,6 +419,23 @@ export class ImmersiveInteractionSystem { native: boolean ) { const intersection = this.raycast(runtime) + if (intersection?.object.userData.statusMenu === true) { + this.options.onStatusToggle('fallback') + return + } + const statusAction = intersection?.object.userData.statusActionId + if (typeof statusAction === 'string') { + this.activateStatusAction( + runtime, + statusAction as StatusActionId, + native ? 'ray' : 'pinch', + intersection!.object + ) + return + } + if (native && this.activateControllerContact(runtime)) { + return + } const action = intersection?.object.userData.action if (action === 'toggle-voice' || action === 'exit-xr') { this.options.onAction(action) @@ -342,6 +471,215 @@ export class ImmersiveInteractionSystem { } } + private activationSource(runtime: SourceRuntime): StatusActivation['source'] { + if (runtime.inputSource?.hand) { + return 'hand' + } + if (runtime.inputSource?.targetRayMode === 'gaze') { + return 'gaze' + } + if (runtime.inputSource?.targetRayMode === 'screen') { + return 'screen' + } + return 'controller' + } + + private activateStatusAction( + runtime: SourceRuntime, + action: StatusActionId, + method: StatusActivation['method'], + target: Object3D + ) { + if (target.userData.statusActionAvailable !== true) { + return false + } + const policy = target.userData.statusInputPolicy as StatusActionInputPolicy | undefined + if (!canActivateStatusAction({ + source: this.activationSource(runtime), + method + }, policy)) { + return false + } + if (this.activationSource(runtime) === 'hand') { + this.statusGesture.suppress(performance.now()) + } + this.options.onStatusAction(action) + return true + } + + private nearestStatusContact(point: Vector3, maximumDistance: number) { + let nearest: { target: Mesh, distance: number } | null = null + for (const target of this.options.getStatusTargets()) { + target.updateWorldMatrix(true, false) + contactBounds.setFromObject(target) + const distance = contactBounds.distanceToPoint(point) + if (distance <= maximumDistance && (!nearest || distance < nearest.distance)) { + nearest = { target, distance } + } + } + return nearest?.target ?? null + } + + private activateControllerContact(runtime: SourceRuntime) { + if ( + !this.options.isStatusOpen() + || runtime.inputSource?.hand + || runtime.inputSource?.targetRayMode !== 'tracked-pointer' + ) { + return false + } + runtime.grip.getWorldPosition(sourcePosition) + const target = this.nearestStatusContact(sourcePosition, 0.055) + const action = target?.userData.statusActionId + return typeof action === 'string' + ? this.activateStatusAction(runtime, action as StatusActionId, 'contact', target!) + : false + } + + private updateHandStatusContact(runtime: SourceRuntime) { + if (!runtime.inputSource?.hand || !this.options.isStatusOpen()) { + runtime.contactActionId = null + return + } + const index = resolveTrackedHandJoint( + runtime.hand, + 'index-finger-tip' + ) + if (!index) { + runtime.contactActionId = null + return + } + index.getWorldPosition(indexPosition) + const target = this.nearestStatusContact(indexPosition, 0.018) + const action = typeof target?.userData.statusActionId === 'string' + ? target.userData.statusActionId as StatusActionId + : null + if (action && runtime.contactActionId !== action) { + this.activateStatusAction(runtime, action, 'contact', target!) + } + runtime.contactActionId = action + } + + private updateMenuButton(runtime: SourceRuntime) { + const source = runtime.inputSource + const index = source + ? mappedStatusMenuButtonIndex(source) + : null + const pressed = index == null + ? false + : source?.gamepad?.buttons[index]?.pressed === true + if (pressed && !runtime.menuPressed) { + this.options.onStatusToggle('controller') + } + runtime.menuPressed = pressed + } + + private updateInputCapabilities() { + const controller = this.sources.some(runtime => + Boolean(runtime.inputSource) + && !runtime.inputSource?.hand + && runtime.inputSource?.targetRayMode === 'tracked-pointer' + ) + const hand = this.sources.some(runtime => + Boolean(runtime.inputSource?.hand) + && resolveTrackedHandJoint(runtime.hand, 'wrist') !== null + ) + const fallbackMenu = resolveStatusFallbackMenuVisibility(this.sources) + const key = `${controller}:${hand}:${fallbackMenu}` + if (key !== this.lastInputCapabilities) { + this.lastInputCapabilities = key + this.options.onInputCapabilitiesChanged({ + controller, + hand, + fallbackMenu + }) + } + } + + private updateStatusGesture(now: number) { + const leftController = this.sources.some(runtime => + runtime.inputSource?.handedness === 'left' + && !runtime.inputSource.hand + && runtime.inputSource.targetRayMode === 'tracked-pointer' + ) + const leftHand = this.sources.find(runtime => + runtime.inputSource?.handedness === 'left' + && Boolean(runtime.inputSource.hand) + ) + const wrist = leftHand + ? resolveTrackedHandJoint(leftHand.hand, 'wrist') + : null + this.options.renderer.xr.getCamera().getWorldPosition(viewerPosition) + let height = Number.NEGATIVE_INFINITY + let facing = Number.NEGATIVE_INFINITY + if (wrist) { + wrist.getWorldPosition(sourcePosition) + wrist.getWorldQuaternion(jointQuaternion) + handBackNormal.set(0, 1, 0).applyQuaternion(jointQuaternion).normalize() + wristToViewer.subVectors(viewerPosition, sourcePosition).normalize() + height = sourcePosition.y - viewerPosition.y + facing = handBackNormal.dot(wristToViewer) + } + const event = this.statusGesture.update({ + now, + tracked: Boolean(wrist), + controllerActive: leftController, + wristHeightFromEyes: height, + handBackFacingViewer: facing + }, { + open: this.options.isStatusOpen(), + invocation: this.options.getStatusInvocation() + }) + if (event === 'open') { + this.options.onStatusToggle('hand') + } else if (event === 'close') { + this.options.onStatusDismiss() + } + } + + private updateControllerArmDismissal(now: number) { + const leftController = this.sources.find(runtime => + runtime.inputSource?.handedness === 'left' + && !runtime.inputSource.hand + && runtime.inputSource.targetRayMode === 'tracked-pointer' + ) + this.options.renderer.xr.getCamera().getWorldPosition(viewerPosition) + let height = Number.NEGATIVE_INFINITY + if (leftController) { + const anchor = leftController.inputSource?.gripSpace + ? leftController.grip + : leftController.targetRay + anchor.getWorldPosition(sourcePosition) + height = sourcePosition.y - viewerPosition.y + } + if (this.statusControllerArm.update({ + now, + tracked: Boolean(leftController), + gripHeightFromEyes: height, + open: this.options.isStatusOpen(), + invocation: this.options.getStatusInvocation() + }) === 'close') { + this.options.onStatusDismiss() + } + } + + statusAnchor() { + const controller = this.sources.find(runtime => + runtime.inputSource?.handedness === 'left' + && !runtime.inputSource.hand + && runtime.inputSource.targetRayMode === 'tracked-pointer' + ) + if (controller) { + return controller.inputSource?.gripSpace ? controller.grip : controller.targetRay + } + const hand = this.sources.find(runtime => + runtime.inputSource?.handedness === 'left' && runtime.inputSource.hand + ) + return hand + ? resolveTrackedHandJoint(hand.hand, 'wrist') + : null + } + private handleGrabStart( runtime: SourceRuntime, activation: NonNullable @@ -380,7 +718,8 @@ export class ImmersiveInteractionSystem { if (!runtime.inputSource?.gripSpace) { runtime.targetRay.getWorldPosition(sourcePosition) } - runtime.grabOffset.copy(panel.group.position).sub(sourcePosition) + panel.group.getWorldPosition(runtime.grabOffset) + runtime.grabOffset.sub(sourcePosition) this.refreshPanelInteraction() } @@ -405,10 +744,7 @@ export class ImmersiveInteractionSystem { .getWorldPosition(viewerPosition) this.options.onPanelFocused( grabbedPanelId, - resolveFocusedPanelPosition( - viewerPosition, - panel.group.position - ) + resolveFocusedPanelLocalPosition(viewerPosition, panel.group) ) } else { this.options.onPanelMoved( @@ -425,9 +761,10 @@ export class ImmersiveInteractionSystem { if (!runtime.inputSource?.hand) { return } - const thumb = runtime.hand.getObjectByName('thumb-tip') - const index = runtime.hand.getObjectByName('index-finger-tip') + const thumb = resolveTrackedHandJoint(runtime.hand, 'thumb-tip') + const index = resolveTrackedHandJoint(runtime.hand, 'index-finger-tip') if (!thumb || !index) { + this.endSynthesizedPinch(runtime, false) return } thumb.getWorldPosition(thumbPosition) @@ -437,12 +774,22 @@ export class ImmersiveInteractionSystem { runtime.pinching = true this.handleSelectStart(runtime, now, false) } else if (runtime.pinching && distance >= 0.038) { - runtime.pinching = false - runtime.selecting = false - this.model.selectEnd(runtime.id) - if (runtime.grabbedBy === 'pinch') { - this.releaseGrab(runtime, true) - } + this.endSynthesizedPinch(runtime, true) + } + } + + private endSynthesizedPinch( + runtime: SourceRuntime, + focusOnTap: boolean + ) { + if (!runtime.pinching) { + return + } + runtime.pinching = false + runtime.selecting = false + this.model.selectEnd(runtime.id) + if (runtime.grabbedBy === 'pinch') { + this.releaseGrab(runtime, focusOnTap) } } @@ -495,6 +842,8 @@ export class ImmersiveInteractionSystem { intersection ? this.hitFromObject(intersection.object) : null ) this.updatePinch(runtime, now) + this.updateHandStatusContact(runtime) + this.updateMenuButton(runtime) this.updateGamepadScroll(runtime) const sourceState = this.model.snapshot().sources.get(runtime.id) @@ -538,6 +887,11 @@ export class ImmersiveInteractionSystem { sourcePosition ) if (position) { + worldPointToPanelLocal( + panel.group, + position, + position + ) runtime.grabMoved ||= !isPanelGrabTap( runtime.grabInitialPosition, position @@ -550,6 +904,11 @@ export class ImmersiveInteractionSystem { runtime.targetRay.getWorldPosition(sourcePosition) } const position = sourcePosition.add(runtime.grabOffset) + worldPointToPanelLocal( + panel.group, + position, + position + ) runtime.grabMoved ||= !isPanelGrabTap( runtime.grabInitialPosition, position @@ -559,6 +918,9 @@ export class ImmersiveInteractionSystem { } } } + this.updateInputCapabilities() + this.updateStatusGesture(now) + this.updateControllerArmDismissal(now) this.refreshPanelInteraction() } diff --git a/packages/webxr/src/main.ts b/packages/webxr/src/main.ts index daeeec97..4d4be42b 100644 --- a/packages/webxr/src/main.ts +++ b/packages/webxr/src/main.ts @@ -4,7 +4,10 @@ import { import type { RealtimeConversationSnapshot } from '@codori/client/shared/realtime' import { detectImmersiveCapability, - requestImmersiveSession + requestImmersiveSession, + resolvePassthroughAvailability, + type ImmersiveModeSupport, + type ImmersiveSessionMode } from './xr-capability' import type { ImmersiveScene } from './immersive-scene' import { @@ -13,7 +16,14 @@ import { } from './voice-runtime' import { coordinateRealtimeAutoStart } from './realtime-auto-start' import { ImmersiveSoundEffects } from './sound-effects' -import { WorkspaceRuntime } from './workspace-runtime' +import { + WorkspaceRuntime, + type WorkspaceRuntimeSnapshot +} from './workspace-runtime' +import { + createStatusActions, + type StatusActionId +} from './status-window-model' import './style.css' const requiredElement = (id: string) => { @@ -38,6 +48,8 @@ const canvas = requiredElement('xr-canvas') const sceneStatus = requiredElement('scene-status') const sceneControls = requiredElement('scene-controls') const exitButton = requiredElement('exit-xr') +const fallbackMenu = requiredElement('fallback-menu') +const domOverlayRoot = requiredElement('app') const route = parseImmersiveWorkspaceRoute(window.location.href) const returnTo = route?.returnTo ?? '/' @@ -47,6 +59,14 @@ const developmentDebug = import.meta.env.DEV && searchParams.get('debug') === '1' const developmentKitchenSink = developmentDebug && searchParams.get('kitchenSink') === '1' +const developmentBlendPreview = developmentKitchenSink + && ( + searchParams.get('blend') === 'alpha-blend' + || searchParams.get('blend') === 'additive' + ) + ? searchParams.get('blend') as 'alpha-blend' | 'additive' + : null +const developmentStatusPreview = searchParams.get('status') !== '0' const soundEffects = new ImmersiveSoundEffects() window.addEventListener('pagehide', () => { void soundEffects.dispose() @@ -60,6 +80,10 @@ let immersiveScenePromise: Promise | null = null let workspaceRuntime: WorkspaceRuntime | null = null let voiceRuntime: VoiceRuntime | null = null let activeSession: XRSession | null = null +let activeSessionMode: ImmersiveSessionMode = 'immersive-vr' +let supportedModes: ImmersiveModeSupport = { vr: false, ar: false } +let defaultEntryMode: ImmersiveSessionMode = 'immersive-vr' +let transitionTarget: ImmersiveSessionMode | null = null let releaseSessionListeners: (() => void) | null = null let releaseWorkspace: (() => void) | null = null let releaseVoice: (() => void) | null = null @@ -67,6 +91,8 @@ let startingRuntime: Promise | null = null let returningTo2d = false let voiceRequested = false let lastWorkspaceError: string | null = null +let latestWorkspace: WorkspaceRuntimeSnapshot | null = null +let latestVoice: RealtimeConversationSnapshot | null = null const disposeConnectedRuntimes = async () => { releaseWorkspace?.() @@ -90,6 +116,50 @@ const sessionActive = (snapshot: RealtimeConversationSnapshot) => || snapshot.state === 'connected' || snapshot.state === 'stopping' +const statusVoiceState = () => { + if (!latestVoice) { + return 'unavailable' as const + } + if (latestVoice.autoplayBlocked) { + return 'resume-audio' as const + } + return sessionActive(latestVoice) ? 'active' as const : 'inactive' as const +} + +const updateStatusWindow = () => { + if (!immersiveScene || !latestWorkspace) { + return + } + const blendMode = activeSession?.environmentBlendMode ?? 'opaque' + const passthrough = resolvePassthroughAvailability({ + arSupported: supportedModes.ar, + vrSupported: supportedModes.vr, + mode: activeSessionMode, + environmentBlendMode: blendMode + }) + immersiveScene.setStatusWindowSnapshot({ + rateLimits: latestWorkspace.rateLimits, + context: latestWorkspace.context, + connection: latestWorkspace.connection, + voice: statusVoiceState(), + activePaneCount: latestWorkspace.panels.length, + threadLabel: latestWorkspace.thread?.preview + || latestWorkspace.thread?.id + || null, + workspaceLabel: route + ? `${route.identity.workspace.kind}:${route.identity.workspace.id}` + : null, + sessionLabel: `${activeSessionMode} · ${blendMode}`, + actions: createStatusActions({ + passthroughSupported: passthrough.supported, + passthroughActive: passthrough.active, + passthroughDisabledReason: passthrough.disabledReason, + voiceState: statusVoiceState(), + reducedEffects: reducedEffects.checked + }) + }) +} + const setEntryMessage = (message: string) => { entryMessage.textContent = message } @@ -115,6 +185,8 @@ const showEntry = () => { } const updateVoiceUi = (snapshot: RealtimeConversationSnapshot) => { + latestVoice = snapshot + updateStatusWindow() immersiveScene?.setTranscript(snapshot.transcripts, snapshot.generation) immersiveScene?.setActivity( snapshot.state === 'error' @@ -178,8 +250,25 @@ const ensureScene = async () => { }, onPanelAppeared: (panelCount) => { soundEffects.playPanelAppear(panelCount) + }, + onStatusAction: (action) => { + void handleStatusAction(action) + }, + onStatusOpened: () => { + soundEffects.playStatusOpen() + }, + onStatusClosed: () => { + soundEffects.playStatusClose() + }, + onStatusFallbackChanged: (visible) => { + fallbackMenu.hidden = !( + visible + && activeSession + && activeSession.domOverlayState + ) } }) + updateStatusWindow() return immersiveScene }) } @@ -203,6 +292,7 @@ const startWorkspaceRuntime = async () => { }) workspaceRuntime = runtime releaseWorkspace = runtime.subscribe((snapshot) => { + latestWorkspace = snapshot lastWorkspaceError = snapshot.error immersiveScene?.setPanels(snapshot.panels) if (!voiceRuntime || !sessionActive(voiceRuntime.getSnapshot())) { @@ -215,6 +305,7 @@ const startWorkspaceRuntime = async () => { if (snapshot.error) { setSceneStatus(snapshot.error, true) } + updateStatusWindow() }) await runtime.start() @@ -263,6 +354,102 @@ const toggleVoice = async () => { } } +const transitionSessionMode = async (mode: ImmersiveSessionMode) => { + const previous = activeSession + if (!previous || transitionTarget) { + return + } + if (mode === 'immersive-ar' && !supportedModes.ar) { + setSceneStatus('This device does not report immersive AR support.', true) + return + } + if (mode === 'immersive-vr' && !supportedModes.vr) { + setSceneStatus('This device does not report immersive VR support.', true) + return + } + transitionTarget = mode + let replacement: XRSession | null = null + try { + await previous.end() + } catch (error) { + transitionTarget = null + setSceneStatus( + `Could not end the current XR session for transition: ${ + error instanceof Error ? error.message : String(error) + }`, + true + ) + return + } + try { + replacement = await requestImmersiveSession({ + secureContext: window.isSecureContext, + xr: navigator.xr + }, mode, domOverlayRoot) + activeSession = replacement + activeSessionMode = mode + bindSessionListeners(replacement) + const scene = await ensureScene() + await scene.setSession(replacement, mode) + workspaceRuntime?.setSuspended(false) + showScene() + updateStatusWindow() + } catch (error) { + if (replacement) { + releaseSessionListeners?.() + releaseSessionListeners = null + if (activeSession === replacement) { + activeSession = null + } + await immersiveScene?.setSession(null).catch(() => {}) + await replacement.end().catch(() => {}) + } + workspaceRuntime?.setSuspended(true) + showEntry() + entryActions.hidden = false + enterButton.hidden = false + retryButton.hidden = true + enterButton.textContent = mode === 'immersive-ar' + ? 'Re-enter passthrough' + : 'Re-enter immersive VR' + enterButton.onclick = () => { + void enterImmersive(mode) + } + setEntryMessage( + `The browser could not switch XR sessions in-place. Your Codori workspace and voice session are preserved; use the explicit re-entry action. ${ + error instanceof Error ? error.message : String(error) + }` + ) + } finally { + transitionTarget = null + } +} + +const handleStatusAction = async (action: StatusActionId) => { + switch (action) { + case 'passthrough': + await transitionSessionMode( + activeSessionMode === 'immersive-ar' + ? 'immersive-vr' + : 'immersive-ar' + ) + break + case 'recenter': + immersiveScene?.recenterWorkspace() + break + case 'voice': + await toggleVoice() + break + case 'reduced-effects': + reducedEffects.checked = !reducedEffects.checked + updateStatusWindow() + break + case 'exit': + await exitImmersive() + break + } +} + const returnTo2d = () => { if (returningTo2d) { return @@ -275,6 +462,10 @@ const handleSessionEnded = () => { releaseSessionListeners?.() releaseSessionListeners = null activeSession = null + fallbackMenu.hidden = true + if (transitionTarget) { + return + } workspaceRuntime?.setSuspended(true) returnTo2d() } @@ -310,7 +501,9 @@ const exitImmersive = async () => { returnTo2d() } -const enterImmersive = async () => { +const enterImmersive = async ( + mode: ImmersiveSessionMode = defaultEntryMode +) => { enterButton.disabled = true retryButton.hidden = true setEntryMessage('Requesting an immersive session…') @@ -319,13 +512,19 @@ const enterImmersive = async () => { const session = await requestImmersiveSession({ secureContext: window.isSecureContext, xr: navigator.xr - }) + }, mode, domOverlayRoot) await soundUnlock activeSession = session + activeSessionMode = mode bindSessionListeners(session) showScene() const scene = await ensureScene() - await scene.setSession(session) + await scene.setSession(session, mode) + if (workspaceRuntime && voiceRuntime) { + workspaceRuntime.setSuspended(false) + updateStatusWindow() + return + } await coordinateRealtimeAutoStart({ prepare: startWorkspaceRuntime, isCurrent: () => activeSession === session, @@ -386,6 +585,53 @@ const enterDebugScene = async () => { scene.setStatus( 'Kitchen sink · non-immersive texture and layout preview' ) + scene.setStatusWindowSnapshot({ + rateLimits: [{ + limitId: 'codex', + limitName: 'Codex', + primary: { + usedPercent: 34, + resetsAt: '2026-08-11T15:00:00+09:00', + windowDurationMins: 300 + }, + secondary: { + usedPercent: 61, + resetsAt: '2026-08-18T09:00:00+09:00', + windowDurationMins: 10_080 + } + }], + context: { + contextWindow: 258_400, + usedTokens: 81_400, + remainingTokens: 177_000, + usedPercent: 31.5, + remainingPercent: 68.5 + }, + connection: 'connected', + voice: 'active', + activePaneCount: fixture.panels.length, + threadLabel: 'Issue #142 kitchen sink', + workspaceLabel: 'project:codori', + sessionLabel: developmentBlendPreview + ? `immersive-ar · ${developmentBlendPreview}` + : 'preview · opaque', + actions: createStatusActions({ + passthroughSupported: false, + passthroughActive: false, + passthroughDisabledReason: 'Preview is not an immersive AR session.', + voiceState: 'active', + reducedEffects: reducedEffects.checked + }) + }) + if (developmentStatusPreview) { + scene.openStatusForPreview() + } + if (developmentBlendPreview) { + canvas.style.background = developmentBlendPreview === 'alpha-blend' + ? 'linear-gradient(135deg, #e9e2cb, #7ea0b0)' + : 'linear-gradient(135deg, #d9c995, #446d82)' + scene.setSessionVisualMode('immersive-ar', developmentBlendPreview) + } return } scene.setStatus( @@ -435,14 +681,20 @@ const checkCapability = async () => { xr: navigator.xr }) if (capability.status === 'available') { + supportedModes = capability.modes + defaultEntryMode = capability.entryMode enterButton.hidden = false - enterButton.textContent = 'Enter immersive Codori' + enterButton.textContent = capability.entryMode === 'immersive-ar' + ? 'Enter immersive AR Codori' + : 'Enter immersive Codori' enterButton.onclick = () => { - void enterImmersive() + void enterImmersive(capability.entryMode) } entryActions.hidden = false setEntryMessage( - 'Your browser reports immersive VR support. Entry and microphone access remain explicit actions.' + capability.entryMode === 'immersive-ar' + ? 'Your browser reports immersive AR support. Transparent pixels will reveal the environment only when the session blend mode permits it.' + : 'Your browser reports immersive VR support. Entry and microphone access remain explicit actions.' ) return } @@ -469,6 +721,10 @@ retryButton.addEventListener('click', () => { exitButton.addEventListener('click', () => { void exitImmersive() }) +fallbackMenu.addEventListener('click', () => { + immersiveScene?.toggleStatusFromFallback() +}) +reducedEffects.addEventListener('change', updateStatusWindow) window.addEventListener('resize', () => { immersiveScene?.resize() }) diff --git a/packages/webxr/src/sound-effect-plans.json b/packages/webxr/src/sound-effect-plans.json index b0afd457..69bd727d 100644 --- a/packages/webxr/src/sound-effect-plans.json +++ b/packages/webxr/src/sound-effect-plans.json @@ -144,5 +144,95 @@ "gain": 0.12 } ] + }, + "statusOpen": { + "durationSeconds": 0.38, + "attackSeconds": 0.008, + "releaseStartSeconds": 0.225, + "peakGain": 0.075, + "echoDelaySeconds": 0, + "echoFeedback": 0, + "echoWetGain": 0, + "tones": [ + { + "wave": "sine", + "delaySeconds": 0, + "startFrequency": 148, + "peakFrequency": 420, + "peakSeconds": 0.38, + "endFrequency": 420, + "gain": 0.46 + }, + { + "wave": "triangle", + "delaySeconds": 0.018, + "attackSeconds": 0.012, + "startFrequency": 222, + "peakFrequency": 632, + "peakSeconds": 0.38, + "endFrequency": 632, + "gain": 0.18 + }, + { + "wave": "sine", + "delaySeconds": 0.052, + "attackSeconds": 0.018, + "startFrequency": 296, + "peakFrequency": 844, + "peakSeconds": 0.38, + "endFrequency": 844, + "gain": 0.1 + } + ] + }, + "statusClose": { + "durationSeconds": 0.3, + "attackSeconds": 0.008, + "releaseStartSeconds": 0.19, + "peakGain": 0.105, + "echoDelaySeconds": 0, + "echoFeedback": 0, + "echoWetGain": 0, + "tones": [ + { + "wave": "sine", + "delaySeconds": 0, + "startFrequency": 430, + "peakFrequency": 260, + "peakSeconds": 0.3, + "endFrequency": 260, + "gain": 0.62 + }, + { + "wave": "triangle", + "delaySeconds": 0.014, + "attackSeconds": 0.012, + "startFrequency": 646, + "peakFrequency": 390, + "peakSeconds": 0.3, + "endFrequency": 390, + "gain": 0.27 + }, + { + "wave": "sine", + "delaySeconds": 0.044, + "attackSeconds": 0.018, + "startFrequency": 884, + "peakFrequency": 520, + "peakSeconds": 0.3, + "endFrequency": 520, + "gain": 0.17 + }, + { + "wave": "sine", + "delaySeconds": 0.058, + "attackSeconds": 0.016, + "startFrequency": 1120, + "peakFrequency": 640, + "peakSeconds": 0.3, + "endFrequency": 640, + "gain": 0.11 + } + ] } } diff --git a/packages/webxr/src/sound-effects.ts b/packages/webxr/src/sound-effects.ts index 13f7b90b..420e1d63 100644 --- a/packages/webxr/src/sound-effects.ts +++ b/packages/webxr/src/sound-effects.ts @@ -27,6 +27,8 @@ export type SoundEffectPlan = { const awakeningPlan = soundEffectPlans.awakening as SoundEffectPlan const panelAppearPlan = soundEffectPlans.panelAppear as SoundEffectPlan +const statusOpenPlan = soundEffectPlans.statusOpen as SoundEffectPlan +const statusClosePlan = soundEffectPlans.statusClose as SoundEffectPlan const easeOutCubic = (progress: number) => 1 - ((1 - progress) ** 3) @@ -72,6 +74,16 @@ export const resolvePanelAppearSoundPlan = ( tones: panelAppearPlan.tones.map(tone => ({ ...tone })) }) +export const resolveStatusOpenSoundPlan = (): SoundEffectPlan => ({ + ...statusOpenPlan, + tones: statusOpenPlan.tones.map(tone => ({ ...tone })) +}) + +export const resolveStatusCloseSoundPlan = (): SoundEffectPlan => ({ + ...statusClosePlan, + tones: statusClosePlan.tones.map(tone => ({ ...tone })) +}) + type AudioContextConstructor = new () => AudioContext const resolveAudioContextConstructor = () => { @@ -237,6 +249,14 @@ export class ImmersiveSoundEffects { return this.play(resolvePanelAppearSoundPlan(panelCount)) } + playStatusOpen() { + return this.play(resolveStatusOpenSoundPlan()) + } + + playStatusClose() { + return this.play(resolveStatusCloseSoundPlan()) + } + async dispose() { const context = this.context this.context = null diff --git a/packages/webxr/src/status-window-model.ts b/packages/webxr/src/status-window-model.ts new file mode 100644 index 00000000..a5fd8d40 --- /dev/null +++ b/packages/webxr/src/status-window-model.ts @@ -0,0 +1,343 @@ +import type { + ContextWindowState, + TokenUsageSnapshot +} from '@codori/client/shared/chat-prompt-controls' +import { + resolveContextWindowState, + shouldShowContextWindowIndicator +} from '@codori/client/shared/chat-prompt-controls' +import { + formatRateLimitWindowDuration, + type RateLimitBucket +} from '@codori/client/shared/account-rate-limits' + +export type StatusActionId = + | 'passthrough' + | 'recenter' + | 'voice' + | 'reduced-effects' + | 'exit' + +export type StatusActionInputPolicy = 'controller-or-touch' | 'any' + +export type StatusAction = { + id: StatusActionId + label: string + state: string | null + available: boolean + disabledReason: string | null + inputPolicy: StatusActionInputPolicy +} + +export type StatusWindowSnapshot = { + rateLimits: RateLimitBucket[] + context: ContextWindowState + connection: string + voice: string + activePaneCount: number + threadLabel: string | null + workspaceLabel: string | null + sessionLabel: string + actions: StatusAction[] +} + +export type StatusQuotaRow = { + id: string + label: string + remainingPercent: number | null + resetsAt: string | null +} + +export const createStatusQuotaRows = ( + buckets: readonly RateLimitBucket[] +): StatusQuotaRow[] => buckets.flatMap(bucket => ([ + bucket.primary + ? { + id: `${bucket.limitId}:primary`, + label: formatRateLimitWindowDuration(bucket.primary.windowDurationMins) + ? `${bucket.limitName ?? bucket.limitId} · ${formatRateLimitWindowDuration(bucket.primary.windowDurationMins)}` + : `${bucket.limitName ?? bucket.limitId} · primary`, + remainingPercent: bucket.primary.usedPercent == null + ? null + : Math.max(0, 100 - bucket.primary.usedPercent), + resetsAt: bucket.primary.resetsAt + } + : null, + bucket.secondary + ? { + id: `${bucket.limitId}:secondary`, + label: formatRateLimitWindowDuration(bucket.secondary.windowDurationMins) + ? `${bucket.limitName ?? bucket.limitId} · ${formatRateLimitWindowDuration(bucket.secondary.windowDurationMins)}` + : `${bucket.limitName ?? bucket.limitId} · secondary`, + remainingPercent: bucket.secondary.usedPercent == null + ? null + : Math.max(0, 100 - bucket.secondary.usedPercent), + resetsAt: bucket.secondary.resetsAt + } + : null +])).filter((row): row is StatusQuotaRow => row !== null) + +export const createStatusActionRowLayout = ( + count: number, + top = 570, + height = 310 +) => Array.from({ length: Math.max(0, count) }, (_, index) => ({ + index, + top: top + ((height / count) * index), + height: height / count +})) + +export const shouldShowStatusFallbackMenu = (input: { + mappedMenuController: boolean + trackedLeftHand: boolean + leftControllerActive: boolean +}) => !input.mappedMenuController && !( + input.trackedLeftHand && !input.leftControllerActive +) + +export const resolveStatusWindowScale = ( + phase: 'opening' | 'open' | 'closing' | 'closed', + progress: number +) => { + const value = Math.min(1, Math.max(0, progress)) + if (phase === 'opening') { + return { x: 0.72 + value * 0.28, y: value } + } + if (phase === 'closing') { + const remaining = 1 - value + return { x: 0.5 + remaining * 0.5, y: remaining } + } + return phase === 'open' + ? { x: 1, y: 1 } + : { x: 0.5, y: 0 } +} + +export const createUnknownContextState = () => + resolveContextWindowState(null, null) + +export const resolveStatusContext = ( + tokenUsage: TokenUsageSnapshot | null, + fallbackContextWindow: number | null = null +) => { + const state = resolveContextWindowState(tokenUsage, fallbackContextWindow) + return { + ...state, + available: shouldShowContextWindowIndicator(state) + } +} + +export type StatusActionState = { + passthroughSupported: boolean + passthroughActive: boolean + passthroughDisabledReason?: string | null + voiceState: 'inactive' | 'active' | 'resume-audio' | 'unavailable' + reducedEffects: boolean +} + +export const createStatusActions = ( + state: StatusActionState +): StatusAction[] => [{ + id: 'passthrough', + label: 'Passthrough', + state: state.passthroughActive ? 'On' : 'Off', + available: state.passthroughSupported, + disabledReason: state.passthroughSupported + ? null + : state.passthroughDisabledReason + ?? 'Immersive AR is not available on this device.', + inputPolicy: 'controller-or-touch' +}, { + id: 'recenter', + label: 'Recenter workspace', + state: null, + available: true, + disabledReason: null, + inputPolicy: 'controller-or-touch' +}, { + id: 'voice', + label: state.voiceState === 'resume-audio' + ? 'Resume audio' + : state.voiceState === 'active' + ? 'Stop voice' + : 'Start voice', + state: state.voiceState === 'active' ? 'On' : 'Off', + available: state.voiceState !== 'unavailable', + disabledReason: state.voiceState === 'unavailable' + ? 'Realtime voice is unavailable.' + : null, + inputPolicy: 'controller-or-touch' +}, { + id: 'reduced-effects', + label: 'Reduced effects', + state: state.reducedEffects ? 'On' : 'Off', + available: true, + disabledReason: null, + inputPolicy: 'controller-or-touch' +}, { + id: 'exit', + label: 'Exit immersive', + state: null, + available: true, + disabledReason: null, + inputPolicy: 'any' +}] + +export type StatusActivation = { + source: 'controller' | 'hand' | 'screen' | 'gaze' + method: 'ray' | 'contact' | 'pinch' +} + +export const canActivateStatusAction = ( + activation: StatusActivation, + policy: StatusActionInputPolicy = 'any' +) => { + if ( + policy === 'controller-or-touch' + && (activation.source === 'screen' || activation.source === 'gaze') + ) { + return false + } + return activation.source === 'hand' + ? activation.method === 'contact' + : activation.method !== 'pinch' +} + +export const mappedMenuButtonIndex = ( + handedness: XRHandedness, + profiles: readonly string[] +) => handedness === 'left' && profiles.includes('htc-vive-focus') ? 4 : null + +export type StatusGestureSample = { + now: number + tracked: boolean + controllerActive: boolean + wristHeightFromEyes: number + handBackFacingViewer: number +} + +export type StatusWindowInvocation = 'controller' | 'hand' | 'fallback' + +export const STATUS_GESTURE_THRESHOLDS = { + openHeightMeters: -0.28, + closeHeightMeters: -0.48, + openFacingDot: 0.55, + closeFacingDot: 0.25, + holdMs: 450, + lowerHoldMs: 180, + cooldownMs: 650 +} as const + +export class StatusGestureModel { + private candidateSince: number | null = null + private lowerSince: number | null = null + private cooldownUntil = 0 + + private trackingLostSince: number | null = null + + suppress(now: number) { + this.candidateSince = null + this.cooldownUntil = now + STATUS_GESTURE_THRESHOLDS.cooldownMs + } + + update( + sample: StatusGestureSample, + state: { open: boolean, invocation: StatusWindowInvocation | null } + ) { + if (sample.controllerActive) { + this.candidateSince = null + this.lowerSince = null + this.trackingLostSince = null + return null + } + if (state.open && state.invocation !== 'hand') { + this.candidateSince = null + this.lowerSince = null + this.trackingLostSince = null + return null + } + if (!sample.tracked) { + this.candidateSince = null + this.lowerSince = null + if (!state.open || state.invocation !== 'hand') { + this.trackingLostSince = null + return null + } + this.trackingLostSince ??= sample.now + return sample.now - this.trackingLostSince >= 300 + ? 'close' as const + : null + } + this.trackingLostSince = null + if (state.open) { + const lowered = sample.wristHeightFromEyes + <= STATUS_GESTURE_THRESHOLDS.closeHeightMeters + || sample.handBackFacingViewer + <= STATUS_GESTURE_THRESHOLDS.closeFacingDot + if (!lowered) { + this.lowerSince = null + return null + } + this.lowerSince ??= sample.now + if (sample.now - this.lowerSince >= STATUS_GESTURE_THRESHOLDS.lowerHoldMs) { + this.lowerSince = null + this.cooldownUntil = sample.now + STATUS_GESTURE_THRESHOLDS.cooldownMs + return 'close' as const + } + return null + } + const posed = sample.wristHeightFromEyes + >= STATUS_GESTURE_THRESHOLDS.openHeightMeters + && sample.handBackFacingViewer + >= STATUS_GESTURE_THRESHOLDS.openFacingDot + if (!posed || sample.now < this.cooldownUntil) { + this.candidateSince = null + return null + } + this.candidateSince ??= sample.now + if (sample.now - this.candidateSince >= STATUS_GESTURE_THRESHOLDS.holdMs) { + this.candidateSince = null + return 'open' as const + } + return null + } +} + +export class StatusControllerArmModel { + private lowerSince: number | null = null + private trackingLostSince: number | null = null + + update(input: { + now: number + tracked: boolean + gripHeightFromEyes: number + open: boolean + invocation: StatusWindowInvocation | null + }) { + if (!input.open || input.invocation !== 'controller') { + this.lowerSince = null + this.trackingLostSince = null + return null + } + if (!input.tracked) { + this.lowerSince = null + this.trackingLostSince ??= input.now + return input.now - this.trackingLostSince >= 300 + ? 'close' as const + : null + } + this.trackingLostSince = null + if (input.gripHeightFromEyes > -0.4) { + this.lowerSince = null + return null + } + if (input.gripHeightFromEyes > -0.55) { + return null + } + this.lowerSince ??= input.now + if (input.now - this.lowerSince >= 250) { + this.lowerSince = null + return 'close' as const + } + return null + } +} diff --git a/packages/webxr/src/status-window-view.ts b/packages/webxr/src/status-window-view.ts new file mode 100644 index 00000000..d3ab1acd --- /dev/null +++ b/packages/webxr/src/status-window-view.ts @@ -0,0 +1,393 @@ +import { + BoxGeometry, + CanvasTexture, + Group, + LinearFilter, + Mesh, + MeshBasicMaterial, + PlaneGeometry, + SRGBColorSpace +} from 'three' +import type { + StatusAction, + StatusWindowSnapshot +} from './status-window-model' +import { + createStatusActionRowLayout, + createStatusQuotaRows, + resolveStatusWindowScale +} from './status-window-model' +import { CanvasTextSurface } from './text-surface' + +const WIDTH_METERS = 0.72 +const HEIGHT_METERS = 0.96 +const WIDTH_PIXELS = 720 +const HEIGHT_PIXELS = 960 +const ACTION_TOP_PIXELS = 570 +const ACTION_AREA_PIXELS = 310 + +const clamp01 = (value: number) => Math.min(1, Math.max(0, value)) + +const formatReset = (value: string | null) => { + if (!value) { + return 'reset unavailable' + } + return new Intl.DateTimeFormat(undefined, { + month: 'short', + day: 'numeric', + hour: 'numeric', + minute: '2-digit' + }).format(new Date(value)) +} + +const progress = ( + context: CanvasRenderingContext2D, + y: number, + label: string, + remainingPercent: number | null, + detail: string +) => { + context.fillStyle = '#dfffaa' + context.font = '600 27px Inter, system-ui, sans-serif' + context.fillText(label, 48, y) + context.textAlign = 'right' + context.fillText( + remainingPercent == null + ? 'Unavailable' + : `${Math.round(remainingPercent)}% remaining`, + WIDTH_PIXELS - 48, + y + ) + context.textAlign = 'left' + context.fillStyle = 'rgba(220, 255, 167, 0.18)' + context.fillRect(48, y + 18, WIDTH_PIXELS - 96, 14) + if (remainingPercent != null) { + context.fillStyle = '#b9f46f' + context.fillRect( + 48, + y + 18, + (WIDTH_PIXELS - 96) * clamp01(remainingPercent / 100), + 14 + ) + } + context.fillStyle = 'rgba(222, 255, 180, 0.66)' + context.font = '22px Inter, system-ui, sans-serif' + context.fillText(detail, 48, y + 62) +} + +export class StatusWindowView { + readonly group = new Group() + + readonly actionHits: Mesh[] = [] + + readonly menuGroup = new Group() + + private readonly windowGroup = new Group() + + readonly menuHit: Mesh + + private readonly canvas = document.createElement('canvas') + + private readonly texture: CanvasTexture + + private readonly surface: Mesh + + private readonly menuSurface = new CanvasTextSurface({ + widthMeters: 0.34, + heightMeters: 0.14, + widthPixels: 520, + heightPixels: 210, + background: 'rgba(52, 74, 18, 0.82)', + border: 'rgba(194, 255, 116, 0.86)', + color: '#e6ffbd', + font: 'Inter, system-ui, sans-serif', + lineHeightPixels: 38, + paddingPixels: 38, + titleFontSizePixels: 52, + glow: true + }) + + private snapshot: StatusWindowSnapshot | null = null + + private phase: 'closed' | 'opening' | 'open' | 'closing' = 'closed' + + private phaseStartedAt = 0 + + private menuRequested = false + + constructor() { + this.group.name = 'status-window' + this.group.visible = false + this.canvas.width = WIDTH_PIXELS + this.canvas.height = HEIGHT_PIXELS + this.texture = new CanvasTexture(this.canvas) + this.texture.colorSpace = SRGBColorSpace + this.texture.minFilter = LinearFilter + this.surface = new Mesh( + new PlaneGeometry(WIDTH_METERS, HEIGHT_METERS), + new MeshBasicMaterial({ + map: this.texture, + transparent: true, + depthWrite: false, + toneMapped: false + }) + ) + this.surface.name = 'status-window-surface' + this.windowGroup.position.y = -HEIGHT_METERS / 2 + this.surface.position.y = HEIGHT_METERS / 2 + this.group.add(this.windowGroup) + this.windowGroup.add(this.surface) + this.reconcileActionHits(0) + + this.menuSurface.render({ title: 'Menu', body: '' }) + this.menuHit = new Mesh( + new BoxGeometry(0.36, 0.16, 0.04), + new MeshBasicMaterial({ transparent: true, opacity: 0, depthWrite: false }) + ) + this.menuHit.userData.statusMenu = true + this.menuGroup.name = 'status-menu-affordance' + this.menuGroup.visible = false + this.menuGroup.add(this.menuSurface.mesh, this.menuHit) + } + + private reconcileActionHits(count: number) { + while (this.actionHits.length > count) { + const hit = this.actionHits.pop()! + hit.removeFromParent() + hit.geometry.dispose() + hit.material.dispose() + } + const rows = createStatusActionRowLayout( + count, + ACTION_TOP_PIXELS, + ACTION_AREA_PIXELS + ) + const rowPixels = rows[0]?.height ?? ACTION_AREA_PIXELS + while (this.actionHits.length < count) { + const index = this.actionHits.length + const hit = new Mesh( + new BoxGeometry(WIDTH_METERS - 0.08, rowPixels / HEIGHT_PIXELS * HEIGHT_METERS, 0.045), + new MeshBasicMaterial({ + transparent: true, + opacity: 0, + depthWrite: false + }) + ) + hit.position.z = 0.025 + hit.name = `status-action-${index}` + this.actionHits.push(hit) + this.windowGroup.add(hit) + } + this.actionHits.forEach((hit, index) => { + const row = rows[index]! + hit.scale.y = rowPixels / ( + (hit.geometry.parameters.height as number) * HEIGHT_PIXELS / HEIGHT_METERS + ) + hit.position.y = (HEIGHT_METERS / 2) + - ((row.top + row.height / 2) / HEIGHT_PIXELS * HEIGHT_METERS) + + (HEIGHT_METERS / 2) + }) + } + + get isOpen() { + return this.phase === 'opening' || this.phase === 'open' + } + + setSnapshot(snapshot: StatusWindowSnapshot) { + this.snapshot = snapshot + this.reconcileActionHits(snapshot.actions.length) + snapshot.actions.forEach((action, index) => { + const hit = this.actionHits[index] + if (hit) { + hit.userData.statusActionId = action.id + hit.userData.statusActionAvailable = action.available + hit.userData.statusInputPolicy = action.inputPolicy + } + }) + this.render() + } + + setMenuVisible(visible: boolean) { + this.menuRequested = visible + this.menuGroup.visible = visible && !this.isOpen + } + + open(now: number) { + if (this.isOpen) { + return false + } + this.phase = 'opening' + this.phaseStartedAt = now + this.group.visible = true + this.menuGroup.visible = false + return true + } + + close(now: number) { + if (this.phase === 'closed' || this.phase === 'closing') { + return false + } + this.phase = 'closing' + this.phaseStartedAt = now + return true + } + + toggle(now: number) { + return this.isOpen ? this.close(now) : this.open(now) + } + + update(now: number, reducedEffects: boolean) { + const duration = reducedEffects ? 1 : this.phase === 'closing' ? 190 : 160 + const progressValue = clamp01((now - this.phaseStartedAt) / duration) + if (this.phase === 'opening') { + const scale = resolveStatusWindowScale('opening', progressValue) + this.windowGroup.scale.set(scale.x, scale.y, 1) + if (progressValue >= 1) { + this.phase = 'open' + } + } else if (this.phase === 'closing') { + // The child group pivots at the lower edge, so this collapses downward. + const scale = resolveStatusWindowScale('closing', progressValue) + this.windowGroup.scale.set(scale.x, scale.y, 1) + if (progressValue >= 1) { + this.phase = 'closed' + this.group.visible = false + this.menuGroup.visible = this.menuRequested + } + } else if (this.phase === 'open') { + this.windowGroup.scale.set(1, 1, 1) + } + } + + private renderAction( + context: CanvasRenderingContext2D, + action: StatusAction, + index: number + ) { + const row = createStatusActionRowLayout( + this.snapshot?.actions.length ?? 0, + ACTION_TOP_PIXELS, + ACTION_AREA_PIXELS + )[index]! + const rowPixels = row.height + const y = row.top + context.fillStyle = action.available + ? 'rgba(177, 235, 96, 0.09)' + : 'rgba(125, 141, 100, 0.05)' + context.fillRect(38, y + 4, WIDTH_PIXELS - 76, rowPixels - 8) + context.fillStyle = action.available + ? '#e3ffb8' + : 'rgba(221, 236, 197, 0.48)' + context.font = '600 27px Inter, system-ui, sans-serif' + context.fillText(action.label, 56, y + Math.min(39, rowPixels * 0.68)) + context.textAlign = 'right' + context.fillText( + action.available ? (action.state ?? '›') : 'Unavailable', + WIDTH_PIXELS - 56, + y + Math.min(39, rowPixels * 0.68) + ) + context.textAlign = 'left' + } + + private render() { + const context = this.canvas.getContext('2d') + const snapshot = this.snapshot + if (!context || !snapshot) { + return + } + context.clearRect(0, 0, WIDTH_PIXELS, HEIGHT_PIXELS) + context.fillStyle = 'rgba(44, 68, 16, 0.78)' + context.fillRect(0, 0, WIDTH_PIXELS, HEIGHT_PIXELS) + context.strokeStyle = 'rgba(194, 255, 116, 0.78)' + context.lineWidth = 5 + context.strokeRect(4, 4, WIDTH_PIXELS - 8, HEIGHT_PIXELS - 8) + context.shadowColor = 'rgba(187, 255, 105, 0.35)' + context.shadowBlur = 18 + context.fillStyle = '#ecffc9' + context.font = '700 42px Inter, system-ui, sans-serif' + context.fillText('Codex status', 48, 68) + context.shadowBlur = 0 + context.fillStyle = 'rgba(222, 255, 180, 0.7)' + context.font = '23px Inter, system-ui, sans-serif' + const identity = [snapshot.workspaceLabel, snapshot.threadLabel] + .filter(Boolean).join(' · ') || 'Workspace identity unavailable' + context.fillText(identity.slice(0, 48), 48, 108) + + const windows = createStatusQuotaRows(snapshot.rateLimits) + const first = windows[0] + const second = windows[1] + if (first) { + progress( + context, + 144, + first.label, + first.remainingPercent, + formatReset(first.resetsAt) + ) + } else { + progress(context, 144, 'Codex quota · primary', null, 'live quota unavailable') + } + progress( + context, + 226, + second?.label ?? 'Codex quota · secondary', + second?.remainingPercent ?? null, + second ? formatReset(second.resetsAt) : 'live quota unavailable' + ) + progress( + context, + 308, + 'Thread context', + snapshot.context.remainingPercent, + snapshot.context.remainingTokens == null + ? 'context usage unavailable' + : `${Math.round(snapshot.context.remainingTokens).toLocaleString()} tokens remaining` + ) + context.fillStyle = 'rgba(222, 255, 180, 0.72)' + context.font = '23px Inter, system-ui, sans-serif' + context.fillText( + `${snapshot.connection} · ${snapshot.voice} · ${snapshot.activePaneCount} panes`, + 48, + 420 + ) + context.fillText(snapshot.sessionLabel, 48, 454) + context.strokeStyle = 'rgba(194, 255, 116, 0.42)' + context.lineWidth = 2 + context.beginPath() + context.moveTo(48, 478) + context.lineTo(WIDTH_PIXELS - 48, 478) + context.stroke() + context.fillStyle = '#dfffaa' + context.font = '700 24px Inter, system-ui, sans-serif' + context.fillText('ACTIONS', 48, 530) + snapshot.actions.forEach((action, index) => { + this.renderAction(context, action, index) + }) + const unavailable = snapshot.actions.find(action => !action.available) + context.fillStyle = 'rgba(222, 255, 180, 0.58)' + context.font = '20px Inter, system-ui, sans-serif' + context.fillText( + (unavailable?.disabledReason ?? 'Touch directly with a tracked index fingertip.').slice(0, 61), + 48, + 928 + ) + this.texture.needsUpdate = true + } + + dispose() { + this.surface.geometry.dispose() + this.surface.material.dispose() + this.texture.dispose() + for (const hit of this.actionHits) { + hit.geometry.dispose() + hit.material.dispose() + } + this.menuHit.geometry.dispose() + this.menuHit.material.dispose() + this.menuSurface.dispose() + this.actionHits.length = 0 + this.group.clear() + this.windowGroup.clear() + this.menuGroup.clear() + } +} diff --git a/packages/webxr/src/workspace-anchor.ts b/packages/webxr/src/workspace-anchor.ts new file mode 100644 index 00000000..4ff8868a --- /dev/null +++ b/packages/webxr/src/workspace-anchor.ts @@ -0,0 +1,53 @@ +import { Quaternion, Vector3 } from 'three' + +export type WorkspaceAnchor = { + position: Vector3 + forward: Vector3 + rotation: Quaternion +} + +export const resolveWorkspaceAnchor = (input: { + viewerPosition: Vector3 + viewerDirection: Vector3 + distanceMeters: number + minimumHeightMeters: number + maximumHeightMeters: number +}): WorkspaceAnchor => { + const forward = input.viewerDirection.clone().setY(0) + if (forward.lengthSq() < 0.001) { + forward.set(0, 0, -1) + } else { + forward.normalize() + } + const position = input.viewerPosition.clone() + .addScaledVector(forward, input.distanceMeters) + position.y = Math.min( + input.maximumHeightMeters, + Math.max(input.minimumHeightMeters, input.viewerPosition.y) + ) + const rotation = new Quaternion().setFromUnitVectors( + new Vector3(0, 0, -1), + forward + ) + return { position, forward, rotation } +} + +export const anchoredWorldPosition = ( + anchor: Vector3, + local: Vector3, + rotation = new Quaternion() +) => local.clone().applyQuaternion(rotation).add(anchor) + +export class ReferenceSpaceResetModel { + private pending = false + + mark() { + this.pending = true + } + + take() { + const pending = this.pending + this.pending = false + return pending + } +} diff --git a/packages/webxr/src/workspace-runtime.ts b/packages/webxr/src/workspace-runtime.ts index 622adf7f..97d35ae4 100644 --- a/packages/webxr/src/workspace-runtime.ts +++ b/packages/webxr/src/workspace-runtime.ts @@ -5,10 +5,22 @@ import { CodexRpcClient } from '@codori/client/shared/codex-rpc' import type { + GetAccountRateLimitsResponse, Thread, ThreadReadResponse, ThreadResumeResponse } from '@codori/client/shared/generated/codex-app-server/v2' +import { + mergeAccountRateLimits, + normalizeAccountRateLimits, + type RateLimitBucket +} from '@codori/client/shared/account-rate-limits' +import { + normalizeThreadTokenUsage, + resolveContextWindowState, + type ContextWindowState, + type TokenUsageSnapshot +} from '@codori/client/shared/chat-prompt-controls' import { listAllThreadBackgroundTerminals, reconcileBackgroundTerminals, @@ -53,6 +65,8 @@ export type WorkspaceRuntimeSnapshot = { panels: SpatialPanelSnapshot[] error: string | null thread: Thread | null + rateLimits: RateLimitBucket[] + context: ContextWindowState } export type WorkspaceRuntimeOptions = { @@ -190,6 +204,10 @@ export class WorkspaceRuntime { private thread: Thread | null = null + private rateLimits: RateLimitBucket[] = [] + + private tokenUsage: TokenUsageSnapshot | null = null + private error: string | null = null private backgroundTerminals: BackgroundTerminalModel[] = [] @@ -237,7 +255,13 @@ export class WorkspaceRuntime { transcripts: [...this.transcriptState.segments], panels: this.panelModel.snapshots(), error: this.error, - thread: this.thread + thread: this.thread, + rateLimits: this.rateLimits.map(bucket => ({ + ...bucket, + primary: bucket.primary ? { ...bucket.primary } : null, + secondary: bucket.secondary ? { ...bucket.secondary } : null + })), + context: resolveContextWindowState(this.tokenUsage, null) } } @@ -285,6 +309,15 @@ export class WorkspaceRuntime { } this.thread = response.thread this.seedRunningItems(response.thread) + try { + const rateLimits = await this.client.request( + 'account/rateLimits/read' + ) + this.rateLimits = normalizeAccountRateLimits(rateLimits) + } catch { + // Quota status is optional; the window remains explicit about unavailable data. + this.rateLimits = [] + } await this.refreshBackgroundTerminals() this.resumeTimers() this.emit() @@ -415,6 +448,15 @@ export class WorkspaceRuntime { this.activity = 'listening' this.error = null break + case 'thread/tokenUsage/updated': + this.tokenUsage = normalizeThreadTokenUsage(notification.params) + break + case 'account/rateLimits/updated': + this.rateLimits = mergeAccountRateLimits( + this.rateLimits, + notification.params + ) + break case 'thread/realtime/transcript/delta': { const role = (notification.params as { role?: unknown }).role this.activity = role === 'assistant' ? 'speaking' : 'transcribing' diff --git a/packages/webxr/src/xr-capability.ts b/packages/webxr/src/xr-capability.ts index 04d744ac..798c8a4e 100644 --- a/packages/webxr/src/xr-capability.ts +++ b/packages/webxr/src/xr-capability.ts @@ -1,5 +1,9 @@ export type ImmersiveCapability = - | { status: 'available' } + | { + status: 'available' + modes: ImmersiveModeSupport + entryMode: ImmersiveSessionMode + } | { status: 'insecure', message: string } | { status: 'unsupported', message: string } | { status: 'failed', message: string } @@ -9,11 +13,87 @@ export type XrCapabilityEnvironment = { xr?: Pick } -export const createImmersiveSessionInit = (): XRSessionInit => ({ +export type ImmersiveSessionMode = 'immersive-vr' | 'immersive-ar' + +export type ImmersiveModeSupport = { + vr: boolean + ar: boolean +} + +export type PassthroughAvailability = { + supported: boolean + active: boolean + contrast: 'dither' | 'additive-shape' | 'opaque' + disabledReason: string | null +} + +export const createImmersiveSessionInit = ( + domOverlayRoot?: Element | null +): XRSessionInit => ({ requiredFeatures: ['local-floor'], - optionalFeatures: ['bounded-floor', 'hand-tracking', 'layers'] + optionalFeatures: [ + 'bounded-floor', + 'hand-tracking', + 'layers', + ...(domOverlayRoot ? ['dom-overlay' as const] : []) + ], + ...(domOverlayRoot + ? { domOverlay: { root: domOverlayRoot } } + : {}) }) +export const resolvePassthroughAvailability = (input: { + arSupported: boolean + vrSupported: boolean + mode: ImmersiveSessionMode + environmentBlendMode: XREnvironmentBlendMode +}): PassthroughAvailability => { + if (!input.arSupported) { + return { + supported: false, + active: false, + contrast: 'opaque', + disabledReason: 'This device does not report immersive AR support.' + } + } + if (input.mode === 'immersive-vr') { + return { + supported: true, + active: false, + contrast: 'opaque', + disabledReason: null + } + } + if (!input.vrSupported) { + return { + supported: false, + active: input.environmentBlendMode !== 'opaque', + contrast: input.environmentBlendMode === 'additive' + ? 'additive-shape' + : input.environmentBlendMode === 'alpha-blend' + ? 'dither' + : 'opaque', + disabledReason: 'Immersive VR is unavailable; exit immersive to leave AR.' + } + } + if (input.environmentBlendMode === 'opaque') { + return { + supported: true, + active: false, + contrast: 'opaque', + disabledReason: null + } + } + return { + supported: true, + active: true, + contrast: input.environmentBlendMode === 'additive' + ? 'additive-shape' + : 'dither', + disabledReason: null + } +} + export const detectImmersiveCapability = async ( environment: XrCapabilityEnvironment ): Promise => { @@ -32,13 +112,34 @@ export const detectImmersiveCapability = async ( } try { - const supported = await environment.xr.isSessionSupported('immersive-vr') - return supported - ? { status: 'available' } - : { - status: 'unsupported', - message: 'This browser does not report support for immersive VR. Continue in the normal Codori workspace.' - } + const [vrProbe, arProbe] = await Promise.allSettled([ + environment.xr.isSessionSupported('immersive-vr'), + environment.xr.isSessionSupported('immersive-ar') + ]) + const vr = vrProbe.status === 'fulfilled' && vrProbe.value + const ar = arProbe.status === 'fulfilled' && arProbe.value + if (vr || ar) { + return { + status: 'available', + modes: { vr, ar }, + entryMode: vr ? 'immersive-vr' : 'immersive-ar' + } + } + if (vrProbe.status === 'rejected' || arProbe.status === 'rejected') { + const error = vrProbe.status === 'rejected' + ? vrProbe.reason + : arProbe.status === 'rejected' + ? arProbe.reason + : 'Unknown WebXR capability error.' + return { + status: 'failed', + message: `Could not check immersive WebXR support: ${error instanceof Error ? error.message : String(error)}` + } + } + return { + status: 'unsupported', + message: 'This browser does not report immersive VR or AR support. Continue in the normal Codori workspace.' + } } catch (error) { return { status: 'failed', @@ -48,7 +149,9 @@ export const detectImmersiveCapability = async ( } export const requestImmersiveSession = async ( - environment: XrCapabilityEnvironment + environment: XrCapabilityEnvironment, + mode: ImmersiveSessionMode = 'immersive-vr', + domOverlayRoot?: Element | null ) => { if (!environment.secureContext) { throw new Error('Immersive Codori requires a secure context.') @@ -58,7 +161,7 @@ export const requestImmersiveSession = async ( } return await environment.xr.requestSession( - 'immersive-vr', - createImmersiveSessionInit() + mode, + createImmersiveSessionInit(domOverlayRoot) ) } diff --git a/packages/webxr/test/billboard.test.ts b/packages/webxr/test/billboard.test.ts index 770de95b..db0f7b3c 100644 --- a/packages/webxr/test/billboard.test.ts +++ b/packages/webxr/test/billboard.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from 'vitest' -import { Vector3 } from 'three' -import { viewerFacingQuaternion } from '../src/billboard' +import { Group, Vector3 } from 'three' +import { + viewerFacingLocalQuaternion, + viewerFacingQuaternion +} from '../src/billboard' describe('viewer-facing billboard', () => { it('points the front face toward the viewer', () => { @@ -16,4 +19,23 @@ describe('viewer-facing billboard', () => { expect(frontDirection.dot(expectedDirection)).toBeCloseTo(1) }) + + it('converts a world-facing target into child-local space under a 90-degree yaw', () => { + const anchor = new Group() + anchor.position.set(2, 0, -1) + anchor.rotation.y = -Math.PI / 2 + const panel = new Group() + panel.position.set(0.4, 1.4, -1.2) + anchor.add(panel) + anchor.updateMatrixWorld(true) + const viewer = new Vector3(0, 1.65, 0) + + panel.quaternion.copy(viewerFacingLocalQuaternion(panel, viewer)) + anchor.updateMatrixWorld(true) + const worldPosition = panel.getWorldPosition(new Vector3()) + const worldQuaternion = panel.getWorldQuaternion(panel.quaternion.clone()) + const front = new Vector3(0, 0, 1).applyQuaternion(worldQuaternion) + const expected = viewer.clone().sub(worldPosition).normalize() + expect(front.dot(expected)).toBeCloseTo(1) + }) }) diff --git a/packages/webxr/test/panel-interaction.test.ts b/packages/webxr/test/panel-interaction.test.ts index 45632889..cbb04303 100644 --- a/packages/webxr/test/panel-interaction.test.ts +++ b/packages/webxr/test/panel-interaction.test.ts @@ -6,14 +6,21 @@ import { Ray, Sphere, Vector3, - type WebGLRenderer + type WebGLRenderer, + type XRHandSpace, + type XRJointSpace } from 'three' import { ImmersiveInteractionSystem, isPanelGrabTap, + mappedStatusMenuButtonIndex, + resolveFocusedPanelLocalPosition, resolveFocusedPanelPosition, resolveRayPanelPosition, - resolveRayGrabPosition + resolveRayGrabPosition, + resolveStatusFallbackMenuVisibility, + worldPointToPanelLocal, + resolveTrackedHandJoint } from '../src/interaction-system' import { PanelInteractionModel } from '../src/panel-interaction' @@ -68,6 +75,26 @@ describe('panel interaction model', () => { expect(resolveFocusedPanelPosition(viewer, close)).toEqual(close) }) + it('keeps focus and drag targets correct under a recentered yaw anchor', () => { + const anchor = new Group() + anchor.position.set(3, 0, -2) + anchor.rotation.y = -Math.PI / 2 + const panel = new Group() + panel.position.set(0.5, 1.4, -2.6) + anchor.add(panel) + anchor.updateMatrixWorld(true) + + const viewer = new Vector3(0, 1.65, 0) + panel.position.copy(resolveFocusedPanelLocalPosition(viewer, panel)) + anchor.updateMatrixWorld(true) + expect(panel.getWorldPosition(new Vector3()).distanceTo(viewer)).toBeCloseTo(1.8) + + const dragWorld = new Vector3(1.2, 1.7, -1.1) + panel.position.copy(worldPointToPanelLocal(panel, dragWorld)) + anchor.updateMatrixWorld(true) + expect(panel.getWorldPosition(new Vector3())).toEqual(dragWorld) + }) + it('tracks content scrolling at the ray intersection instead of controller height', () => { const panel = new Group() panel.position.set(0, 1, -2) @@ -152,6 +179,209 @@ describe('panel interaction model', () => { expect(model.selectStart('hand', hit, 1_300, false)).toBe(true) }) + it('reads only currently tracked Three.js hand joints from the joints map', () => { + const hand = Object.assign(new Group(), { + joints: {}, + inputState: { pinching: false } + }) as unknown as XRHandSpace + const wrist = Object.assign(new Group(), { + jointRadius: 0.01 + }) as unknown as XRJointSpace + hand.joints.wrist = wrist + hand.visible = true + wrist.visible = true + + expect(wrist.name).toBe('') + expect(hand.getObjectByName('wrist')).toBeUndefined() + expect(resolveTrackedHandJoint(hand, 'wrist')).toBe(wrist) + + wrist.visible = false + expect(resolveTrackedHandJoint(hand, 'wrist')).toBe(null) + wrist.visible = true + hand.visible = false + expect(resolveTrackedHandJoint(hand, 'wrist')).toBe(null) + }) + + it('recognizes a menu controller only when its mapped button is exposed', () => { + const source = { + handedness: 'left', + profiles: ['htc-vive-focus'], + gamepad: { + buttons: [{}, {}, {}, {}, { pressed: false }] + } + } as unknown as Pick + expect(mappedStatusMenuButtonIndex(source)).toBe(4) + expect(mappedStatusMenuButtonIndex({ + ...source, + gamepad: { buttons: [{}, {}, {}, {}] } as unknown as Gamepad + })).toBe(null) + expect(mappedStatusMenuButtonIndex({ + ...source, + profiles: ['unknown-controller'] + })).toBe(null) + }) + + it('shows the fallback only when the connected sources have no status invocation path', () => { + const handSource = (handedness: XRHandedness) => { + const hand = Object.assign(new Group(), { + joints: {}, + inputState: { pinching: false } + }) as unknown as XRHandSpace + const wrist = Object.assign(new Group(), { + jointRadius: 0.01 + }) as unknown as XRJointSpace + hand.joints.wrist = wrist + hand.visible = true + wrist.visible = true + return { + inputSource: { + handedness, + hand: {}, + targetRayMode: 'tracked-pointer', + profiles: [] + } as unknown as XRInputSource, + hand + } + } + const controllerSource = ( + handedness: XRHandedness, + mapped: boolean + ) => ({ + inputSource: { + handedness, + hand: null, + targetRayMode: 'tracked-pointer', + profiles: mapped ? ['htc-vive-focus'] : ['unknown-controller'], + gamepad: { + buttons: [{}, {}, {}, {}, { pressed: false }] + } + } as unknown as XRInputSource, + hand: Object.assign(new Group(), { + joints: {}, + inputState: { pinching: false } + }) as unknown as XRHandSpace + }) + + const rightHand = handSource('right') + const leftHand = handSource('left') + const leftMappedController = controllerSource('left', true) + const leftUnmappedController = controllerSource('left', false) + const rightUnmappedController = controllerSource('right', false) + + expect(resolveStatusFallbackMenuVisibility([rightHand])).toBe(true) + expect(resolveStatusFallbackMenuVisibility([leftHand])).toBe(false) + expect(resolveStatusFallbackMenuVisibility([ + leftMappedController + ])).toBe(false) + expect(resolveStatusFallbackMenuVisibility([ + leftHand, + leftUnmappedController + ])).toBe(true) + expect(resolveStatusFallbackMenuVisibility([ + leftHand, + rightUnmappedController + ])).toBe(false) + expect(resolveStatusFallbackMenuVisibility([ + rightHand, + leftMappedController + ])).toBe(false) + }) + + it.each(['thumb-tip', 'index-finger-tip'] as const)( + 'ends an active synthesized pinch once when %s tracking is lost', + (lostJointName) => { + const targetRays = [new Group(), new Group()] + const grips = [new Group(), new Group()] + const hands = [0, 1].map(() => Object.assign(new Group(), { + joints: {}, + inputState: { pinching: false } + }) as unknown as XRHandSpace) + const renderer = { + xr: { + getController: (index: number) => targetRays[index], + getControllerGrip: (index: number) => grips[index], + getHand: (index: number) => hands[index] + } + } as unknown as WebGLRenderer + const system = new ImmersiveInteractionSystem({ + renderer, + root: new Group(), + getPanels: () => new Map(), + getControlTargets: () => [], + getStatusTargets: () => [], + getStatusMenuTarget: () => null, + isStatusOpen: () => false, + getStatusInvocation: () => null, + onScroll: () => {}, + onPanelInteracted: () => {}, + onPanelMoved: () => {}, + onPanelFocused: () => {}, + onPanelDismiss: () => {}, + onAction: () => {}, + onStatusToggle: () => {}, + onStatusDismiss: () => {}, + onStatusAction: () => {}, + onInputCapabilitiesChanged: () => {} + }) + type TestRuntime = { + id: string + hand: XRHandSpace + inputSource: XRInputSource | null + pinching: boolean + selecting: boolean + grabbedBy: 'select' | 'squeeze' | 'pinch' | null + } + const internals = system as unknown as { + sources: TestRuntime[] + model: PanelInteractionModel + updatePinch: (runtime: TestRuntime, now: number) => void + } + const runtime = internals.sources[0]! + runtime.inputSource = { + handedness: 'left', + hand: {}, + targetRayMode: 'tracked-pointer', + profiles: [] + } as unknown as XRInputSource + runtime.hand.visible = true + for (const name of ['thumb-tip', 'index-finger-tip'] as const) { + const joint = Object.assign(new Group(), { + jointRadius: 0.01 + }) as unknown as XRJointSpace + joint.visible = name !== lostJointName + runtime.hand.joints[name] = joint + } + const hit = { panelId: 'panel-1', zone: 'grab' as const } + internals.model.selectStart(runtime.id, hit, 0, false) + internals.model.grabStart(runtime.id, hit) + runtime.pinching = true + runtime.selecting = true + runtime.grabbedBy = 'pinch' + const selectEnd = vi.spyOn(internals.model, 'selectEnd') + const releaseGrab = vi.spyOn(internals.model, 'releaseGrab') + + internals.updatePinch(runtime, 100) + + expect(runtime).toMatchObject({ + pinching: false, + selecting: false, + grabbedBy: null + }) + expect(internals.model.snapshot().sources.get(runtime.id)).toMatchObject({ + selected: null, + grabbedPanelId: null + }) + expect(internals.model.snapshot().grabOwners).toHaveLength(0) + expect(selectEnd).toHaveBeenCalledTimes(1) + expect(releaseGrab).toHaveBeenCalledTimes(1) + + internals.updatePinch(runtime, 101) + expect(selectEnd).toHaveBeenCalledTimes(1) + expect(releaseGrab).toHaveBeenCalledTimes(1) + system.dispose() + } + ) + it('removes input listeners and disposes fallback geometry on teardown', () => { const targetRays = [new Group(), new Group()] const grips = [new Group(), new Group()] @@ -169,12 +399,20 @@ describe('panel interaction model', () => { root, getPanels: () => new Map(), getControlTargets: () => [], + getStatusTargets: () => [], + getStatusMenuTarget: () => null, + isStatusOpen: () => false, + getStatusInvocation: () => null, onScroll: () => {}, onPanelInteracted: () => {}, onPanelMoved: () => {}, onPanelFocused: () => {}, onPanelDismiss: () => {}, - onAction: () => {} + onAction: () => {}, + onStatusToggle: () => {}, + onStatusDismiss: () => {}, + onStatusAction: () => {}, + onInputCapabilitiesChanged: () => {} }) const listenerRemoval = targetRays.map(targetRay => vi.spyOn(targetRay, 'removeEventListener') diff --git a/packages/webxr/test/sound-effects.test.ts b/packages/webxr/test/sound-effects.test.ts index 52a06f54..a408a619 100644 --- a/packages/webxr/test/sound-effects.test.ts +++ b/packages/webxr/test/sound-effects.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from 'vitest' import { resolveAwakeningSoundPlan, - resolvePanelAppearSoundPlan + resolvePanelAppearSoundPlan, + resolveStatusCloseSoundPlan, + resolveStatusOpenSoundPlan } from '../src/sound-effects' describe('immersive sound effects', () => { @@ -47,4 +49,15 @@ describe('immersive sound effects', () => { expect(grouped.peakGain).toBeGreaterThan(single.peakGain) expect(grouped.peakGain).toBeLessThanOrEqual(single.peakGain * 1.5) }) + + it('uses longer mirrored pitch-up and pitch-down status cues', () => { + const open = resolveStatusOpenSoundPlan() + const close = resolveStatusCloseSoundPlan() + expect(open.durationSeconds).toBe(0.38) + expect(close.durationSeconds).toBe(0.3) + expect(open.durationSeconds).toBeGreaterThan(0.25) + expect(open.tones.every(tone => tone.endFrequency > tone.startFrequency)).toBe(true) + expect(close.tones.every(tone => tone.endFrequency < tone.startFrequency)).toBe(true) + expect(Math.min(...close.tones.map(tone => tone.endFrequency))).toBeGreaterThanOrEqual(260) + }) }) diff --git a/packages/webxr/test/status-window-model.test.ts b/packages/webxr/test/status-window-model.test.ts new file mode 100644 index 00000000..556a66e8 --- /dev/null +++ b/packages/webxr/test/status-window-model.test.ts @@ -0,0 +1,250 @@ +import { describe, expect, it } from 'vitest' +import { + mergeAccountRateLimits, + normalizeAccountRateLimits +} from '@codori/client/shared/account-rate-limits' +import { + canActivateStatusAction, + createStatusActions, + createStatusActionRowLayout, + createStatusQuotaRows, + mappedMenuButtonIndex, + resolveStatusContext, + resolveStatusWindowScale, + shouldShowStatusFallbackMenu, + StatusControllerArmModel, + StatusGestureModel +} from '../src/status-window-model' + +describe('XR status window model', () => { + it('merges a singular sparse rate-limit update without losing other buckets or fields', () => { + const initial = normalizeAccountRateLimits({ + rateLimits: { + limitId: 'codex', + limitName: 'Codex', + primary: { + usedPercent: 20, + resetsAt: '2026-08-11T12:00:00Z', + windowDurationMins: 300 + }, + secondary: null + }, + rateLimitsByLimitId: { + review: { + limitId: 'review', + limitName: 'Review', + primary: { usedPercent: 40 }, + secondary: null + } + } + }) + + expect(mergeAccountRateLimits(initial, { + rateLimits: { + limitId: 'codex', + limitName: null, + primary: { usedPercent: 35 }, + secondary: null + } + })).toEqual([ + expect.objectContaining({ + limitId: 'codex', + limitName: 'Codex', + primary: { + usedPercent: 35, + resetsAt: '2026-08-11T12:00:00Z', + windowDurationMins: 300 + } + }), + expect.objectContaining({ limitId: 'review' }) + ]) + }) + + it('keeps context unavailable until both occupancy and window are known', () => { + expect(resolveStatusContext(null)).toMatchObject({ + available: false, + remainingPercent: null + }) + expect(resolveStatusContext({ + totalTokens: 10, + totalInputTokens: 5, + totalCachedInputTokens: 0, + totalOutputTokens: 5, + lastUsageKnown: true, + lastTotalTokens: 2_000, + lastInputTokens: 1_500, + lastCachedInputTokens: 0, + lastOutputTokens: 500, + modelContextWindow: 10_000 + })).toMatchObject({ + available: true, + remainingPercent: 80, + remainingTokens: 8_000 + }) + }) + + it('keeps both authoritative quota windows and their unknown states', () => { + expect(createStatusQuotaRows([{ + limitId: 'codex', + limitName: 'Codex', + primary: { usedPercent: 25, resetsAt: null, windowDurationMins: 300 }, + secondary: { usedPercent: null, resetsAt: '2026-08-18T00:00:00Z', windowDurationMins: 10_080 } + }])).toEqual([{ + id: 'codex:primary', + label: 'Codex · 5h window', + remainingPercent: 75, + resetsAt: null + }, { + id: 'codex:secondary', + label: 'Codex · 1w window', + remainingPercent: null, + resetsAt: '2026-08-18T00:00:00Z' + }]) + }) + + it('builds the complete action registry with honest availability', () => { + expect(createStatusActions({ + passthroughSupported: false, + passthroughActive: false, + passthroughDisabledReason: 'The current XR session is opaque.', + voiceState: 'resume-audio', + reducedEffects: true + })).toEqual([ + expect.objectContaining({ id: 'passthrough', available: false, state: 'Off' }), + expect.objectContaining({ id: 'recenter' }), + expect.objectContaining({ id: 'voice', label: 'Resume audio' }), + expect.objectContaining({ id: 'reduced-effects', state: 'On' }), + expect.objectContaining({ id: 'exit' }) + ]) + }) + + it('lays out future action rows without a fixed five-action hit contract', () => { + const rows = createStatusActionRowLayout(8) + expect(rows).toHaveLength(8) + expect(rows[0]?.top).toBe(570) + expect(rows.at(-1)!.top + rows.at(-1)!.height).toBeCloseTo(880) + expect(rows.every((row, index) => + index === 0 || row.top >= rows[index - 1]!.top + rows[index - 1]!.height + )).toBe(true) + }) + + it('narrows the window into its lower-edge pivot on dismissal', () => { + expect(resolveStatusWindowScale('closing', 0)).toEqual({ x: 1, y: 1 }) + expect(resolveStatusWindowScale('closing', 0.5)).toEqual({ x: 0.75, y: 0.5 }) + expect(resolveStatusWindowScale('closing', 1)).toEqual({ x: 0.5, y: 0 }) + }) + + it('allows hand actions only through direct fingertip contact', () => { + expect(canActivateStatusAction({ source: 'hand', method: 'contact' })).toBe(true) + expect(canActivateStatusAction({ source: 'hand', method: 'ray' })).toBe(false) + expect(canActivateStatusAction({ source: 'hand', method: 'pinch' })).toBe(false) + expect(canActivateStatusAction({ source: 'controller', method: 'ray' })).toBe(true) + expect(canActivateStatusAction({ source: 'controller', method: 'contact' })).toBe(true) + expect(canActivateStatusAction( + { source: 'gaze', method: 'ray' }, + 'controller-or-touch' + )).toBe(false) + }) + + it('uses only the verified HTC Vive Focus menu component mapping', () => { + expect(mappedMenuButtonIndex('left', ['htc-vive-focus'])).toBe(4) + expect(mappedMenuButtonIndex('right', ['htc-vive-focus'])).toBe(null) + expect(mappedMenuButtonIndex('left', ['htc-vive'])).toBe(null) + expect(mappedMenuButtonIndex('left', ['unknown-extra-buttons'])).toBe(null) + }) + + it('shows the fallback exactly when no mapped menu or eligible left-hand gesture exists', () => { + // A visible right hand cannot invoke the left-hand status gesture. + expect(shouldShowStatusFallbackMenu({ + mappedMenuController: false, + trackedLeftHand: false, + leftControllerActive: false + })).toBe(true) + // A visible left hand can invoke the gesture when no left controller wins. + expect(shouldShowStatusFallbackMenu({ + mappedMenuController: false, + trackedLeftHand: true, + leftControllerActive: false + })).toBe(false) + // A mapped left controller provides the invocation path. + expect(shouldShowStatusFallbackMenu({ + mappedMenuController: true, + trackedLeftHand: false, + leftControllerActive: true + })).toBe(false) + // An unmapped left controller suppresses the left-hand gesture, so its ray + // must retain the fallback target. + expect(shouldShowStatusFallbackMenu({ + mappedMenuController: false, + trackedLeftHand: true, + leftControllerActive: true + })).toBe(true) + // A right controller does not suppress an eligible left hand. + expect(shouldShowStatusFallbackMenu({ + mappedMenuController: false, + trackedLeftHand: true, + leftControllerActive: false + })).toBe(false) + }) + + it('debounces the hand-back pose with hysteresis and cooldown', () => { + const gesture = new StatusGestureModel() + const posed = { + tracked: true, + controllerActive: false, + wristHeightFromEyes: -0.2, + handBackFacingViewer: 0.7 + } + const closed = { open: false, invocation: null } + expect(gesture.update({ ...posed, now: 0 }, closed)).toBe(null) + expect(gesture.update({ ...posed, now: 449 }, closed)).toBe(null) + expect(gesture.update({ ...posed, now: 450 }, closed)).toBe('open') + const lowered = { + ...posed, + wristHeightFromEyes: -0.6, + handBackFacingViewer: 0.1 + } + const handOpen = { open: true, invocation: 'hand' as const } + expect(gesture.update({ ...lowered, now: 500 }, handOpen)).toBe(null) + expect(gesture.update({ ...lowered, now: 680 }, handOpen)).toBe('close') + expect(gesture.update({ ...posed, now: 1_000 }, closed)).toBe(null) + }) + + it('does not let the hand gesture close a controller-opened window', () => { + const gesture = new StatusGestureModel() + expect(gesture.update({ + now: 1_000, + tracked: false, + controllerActive: true, + wristHeightFromEyes: -1, + handBackFacingViewer: -1 + }, { open: true, invocation: 'controller' })).toBe(null) + }) + + it('closes controller-opened UI only after a lowered hold or tracking-loss grace', () => { + const arm = new StatusControllerArmModel() + const state = { open: true, invocation: 'controller' as const } + expect(arm.update({ ...state, now: 0, tracked: true, gripHeightFromEyes: -0.6 })).toBe(null) + expect(arm.update({ ...state, now: 249, tracked: true, gripHeightFromEyes: -0.6 })).toBe(null) + expect(arm.update({ ...state, now: 250, tracked: true, gripHeightFromEyes: -0.6 })).toBe('close') + + const lost = new StatusControllerArmModel() + expect(lost.update({ ...state, now: 1_000, tracked: false, gripHeightFromEyes: -1 })).toBe(null) + expect(lost.update({ ...state, now: 1_299, tracked: false, gripHeightFromEyes: -1 })).toBe(null) + expect(lost.update({ ...state, now: 1_300, tracked: false, gripHeightFromEyes: -1 })).toBe('close') + }) + + it('gives hand tracking loss a grace period before dismissal', () => { + const gesture = new StatusGestureModel() + const state = { open: true, invocation: 'hand' as const } + const sample = { + tracked: false, + controllerActive: false, + wristHeightFromEyes: -1, + handBackFacingViewer: -1 + } + expect(gesture.update({ ...sample, now: 0 }, state)).toBe(null) + expect(gesture.update({ ...sample, now: 299 }, state)).toBe(null) + expect(gesture.update({ ...sample, now: 300 }, state)).toBe('close') + }) +}) diff --git a/packages/webxr/test/workspace-anchor.test.ts b/packages/webxr/test/workspace-anchor.test.ts new file mode 100644 index 00000000..a0f80471 --- /dev/null +++ b/packages/webxr/test/workspace-anchor.test.ts @@ -0,0 +1,69 @@ +import { describe, expect, it } from 'vitest' +import { Vector3 } from 'three' +import { + anchoredWorldPosition, + ReferenceSpaceResetModel, + resolveWorkspaceAnchor +} from '../src/workspace-anchor' + +describe('XR workspace anchor', () => { + it('recenters in horizontal gaze while clamping eye height', () => { + const anchor = resolveWorkspaceAnchor({ + viewerPosition: new Vector3(2, 2.4, 3), + viewerDirection: new Vector3(1, -0.4, -1), + distanceMeters: 1.45, + minimumHeightMeters: 1.1, + maximumHeightMeters: 1.9 + }) + expect(anchor.position.y).toBe(1.9) + expect(anchor.forward.y).toBe(0) + expect(anchor.position.distanceTo(new Vector3(2, 1.9, 3))).toBeCloseTo(1.45) + }) + + it('rotates the workspace basis into a 90-degree gaze while preserving local transforms', () => { + const anchor = resolveWorkspaceAnchor({ + viewerPosition: new Vector3(0, 1.65, 0), + viewerDirection: new Vector3(1, 0, 0), + distanceMeters: 1.5, + minimumHeightMeters: 1, + maximumHeightMeters: 2 + }) + const leftPane = anchoredWorldPosition( + anchor.position, + new Vector3(-0.5, 0, -0.4), + anchor.rotation + ) + const rightPane = anchoredWorldPosition( + anchor.position, + new Vector3(0.5, 0, -0.4), + anchor.rotation + ) + expect(anchor.forward).toEqual(new Vector3(1, 0, 0)) + expect(leftPane.distanceTo(rightPane)).toBeCloseTo(1) + expect(leftPane.x).toBeCloseTo(rightPane.x) + expect(leftPane.z).not.toBeCloseTo(rightPane.z) + }) + + it('moves every local pane by one anchor delta without changing identity or spacing', () => { + const local = new Map([ + ['pane-a', new Vector3(-0.4, 1.3, -0.2)], + ['pane-b', new Vector3(0.5, 1.6, -0.4)] + ]) + const beforeAnchor = new Vector3(0, 0, 0) + const afterAnchor = new Vector3(3, 0, -2) + const before = [...local].map(([id, point]) => [id, anchoredWorldPosition(beforeAnchor, point)] as const) + const after = [...local].map(([id, point]) => [id, anchoredWorldPosition(afterAnchor, point)] as const) + expect(after.map(([id]) => id)).toEqual(before.map(([id]) => id)) + expect(after[0]![1].distanceTo(after[1]![1])).toBeCloseTo( + before[0]![1].distanceTo(before[1]![1]) + ) + }) + + it('coalesces a reference-space reset into exactly one recenter', () => { + const reset = new ReferenceSpaceResetModel() + reset.mark() + reset.mark() + expect(reset.take()).toBe(true) + expect(reset.take()).toBe(false) + }) +}) diff --git a/packages/webxr/test/workspace-runtime.test.ts b/packages/webxr/test/workspace-runtime.test.ts index a233d221..055724c9 100644 --- a/packages/webxr/test/workspace-runtime.test.ts +++ b/packages/webxr/test/workspace-runtime.test.ts @@ -38,6 +38,17 @@ describe('immersive workspace runtime', () => { if (method === 'thread/read') { return { thread } } + if (method === 'account/rateLimits/read') { + return { + rateLimits: { + limitId: 'codex', + limitName: 'Codex', + primary: { usedPercent: 20 }, + secondary: null + }, + rateLimitsByLimitId: null + } + } if (method === 'thread/backgroundTerminals/list') { return { data: [], @@ -72,6 +83,7 @@ describe('immersive workspace runtime', () => { expect(requests).toEqual([ 'thread/resume', 'thread/read', + 'account/rateLimits/read', 'thread/backgroundTerminals/list' ]) expect(runtime.snapshot()).toMatchObject({ @@ -136,6 +148,9 @@ describe('immersive workspace runtime', () => { nextCursor: null } } + if (method === 'account/rateLimits/read') { + return { rateLimits: [], rateLimitsByLimitId: null } + } throw new Error(`Unexpected request: ${method}`) }) } as unknown as CodexRpcClient @@ -288,4 +303,85 @@ describe('immersive workspace runtime', () => { }) await runtime.dispose() }) + + it('tracks active-thread context and singular sparse quota updates', async () => { + const threadId = 'thread-142' + const thread = { id: threadId, ephemeral: false, turns: [] } as unknown as Thread + let notify: (notification: CodexRpcNotification) => void = () => {} + const client = { + connect: vi.fn(async () => {}), + close: vi.fn(), + subscribe: vi.fn((listener: typeof notify) => { + notify = listener + return () => {} + }), + subscribeConnectionState: vi.fn(() => () => {}), + request: vi.fn(async (method: string) => { + if (method === 'thread/resume' || method === 'thread/read') return { thread } + if (method === 'account/rateLimits/read') { + return { + rateLimits: { + limitId: 'codex', + limitName: 'Codex', + primary: { usedPercent: 10, resetsAt: '2026-08-12T00:00:00Z' }, + secondary: null + }, + rateLimitsByLimitId: null + } + } + if (method === 'thread/backgroundTerminals/list') return { data: [], nextCursor: null } + throw new Error(`Unexpected request: ${method}`) + }) + } as unknown as CodexRpcClient + const runtime = new WorkspaceRuntime({ + identity: { workspace: { kind: 'project', id: 'codori' }, threadId }, + client, + setInterval: vi.fn(() => 1) as unknown as typeof globalThis.setInterval, + clearInterval: vi.fn() as unknown as typeof globalThis.clearInterval + }) + await runtime.start() + + notify({ + method: 'thread/tokenUsage/updated', + params: { + threadId: 'another-thread', + turnId: 'turn-other', + tokenUsage: { total: {}, last: { totalTokens: 9_000 }, modelContextWindow: 10_000 } + } + } as CodexRpcNotification) + expect(runtime.snapshot().context.remainingPercent).toBe(null) + + notify({ + method: 'thread/tokenUsage/updated', + params: { + threadId, + turnId: 'turn-1', + tokenUsage: { total: {}, last: { totalTokens: 2_500 }, modelContextWindow: 10_000 } + } + } as CodexRpcNotification) + notify({ + method: 'account/rateLimits/updated', + params: { + rateLimits: { + limitId: 'codex', + limitName: null, + primary: { usedPercent: 25 }, + secondary: null + } + } + } as CodexRpcNotification) + + expect(runtime.snapshot()).toMatchObject({ + context: { remainingPercent: 75, remainingTokens: 7_500 }, + rateLimits: [{ + limitId: 'codex', + limitName: 'Codex', + primary: { + usedPercent: 25, + resetsAt: '2026-08-12T00:00:00Z' + } + }] + }) + await runtime.dispose() + }) }) diff --git a/packages/webxr/test/world-controls.test.ts b/packages/webxr/test/world-controls.test.ts index d010212c..7a7b3192 100644 --- a/packages/webxr/test/world-controls.test.ts +++ b/packages/webxr/test/world-controls.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest' -import { Vector3 } from 'three' +import { Quaternion, Vector3 } from 'three' import { resolveExitDoorPosition } from '../src/world-controls' describe('world exit door placement', () => { @@ -27,4 +27,24 @@ describe('world exit door placement', () => { expect(Math.min(Math.abs(offset.x), Math.abs(offset.z))) .toBeGreaterThan(0.9) }) + + it('lays out the exit once in anchor-local space at 90-degree workspace yaw', () => { + const localCenter = new Vector3(0, 1.65, 0) + const localDoor = resolveExitDoorPosition( + localCenter, + new Vector3(0, 0, -1) + ) + const yaw = new Quaternion().setFromUnitVectors( + new Vector3(0, 0, -1), + new Vector3(1, 0, 0) + ) + const anchorPosition = new Vector3(1.5, 0, 0) + const worldDoor = localDoor.clone().applyQuaternion(yaw).add(anchorPosition) + const recoveredLocal = worldDoor.clone().sub(anchorPosition) + .applyQuaternion(yaw.clone().invert()) + + expect(recoveredLocal.distanceTo(localDoor)).toBeLessThan(1e-9) + expect(Math.max(Math.abs(recoveredLocal.x), Math.abs(recoveredLocal.z))) + .toBeCloseTo(4.965) + }) }) diff --git a/packages/webxr/test/xr-capability.test.ts b/packages/webxr/test/xr-capability.test.ts index 5e43785f..4d3d014e 100644 --- a/packages/webxr/test/xr-capability.test.ts +++ b/packages/webxr/test/xr-capability.test.ts @@ -2,7 +2,8 @@ import { describe, expect, it, vi } from 'vitest' import { createImmersiveSessionInit, detectImmersiveCapability, - requestImmersiveSession + requestImmersiveSession, + resolvePassthroughAvailability } from '../src/xr-capability' describe('immersive WebXR capability', () => { @@ -20,11 +21,61 @@ describe('immersive WebXR capability', () => { await expect(detectImmersiveCapability({ secureContext: true, xr: { isSessionSupported, requestSession } - })).resolves.toEqual({ status: 'available' }) + })).resolves.toEqual({ + status: 'available', + modes: { vr: true, ar: true }, + entryMode: 'immersive-vr' + }) expect(isSessionSupported).toHaveBeenCalledWith('immersive-vr') + expect(isSessionSupported).toHaveBeenCalledWith('immersive-ar') expect(requestSession).not.toHaveBeenCalled() }) + it.each([ + { vr: true, ar: false, entryMode: 'immersive-vr' }, + { vr: false, ar: true, entryMode: 'immersive-ar' }, + { vr: true, ar: true, entryMode: 'immersive-vr' } + ] as const)('supports the $entryMode capability matrix', async ({ vr, ar, entryMode }) => { + const isSessionSupported = vi.fn(async (mode: XRSessionMode) => + mode === 'immersive-vr' ? vr : ar + ) + await expect(detectImmersiveCapability({ + secureContext: true, + xr: { isSessionSupported, requestSession: vi.fn() } + })).resolves.toMatchObject({ + status: 'available', + modes: { vr, ar }, + entryMode + }) + }) + + it('reports neither mode as unsupported', async () => { + await expect(detectImmersiveCapability({ + secureContext: true, + xr: { + isSessionSupported: vi.fn(async () => false), + requestSession: vi.fn() + } + })).resolves.toMatchObject({ status: 'unsupported' }) + }) + + it('keeps working VR available when the AR probe rejects', async () => { + await expect(detectImmersiveCapability({ + secureContext: true, + xr: { + isSessionSupported: vi.fn(async (mode: XRSessionMode) => { + if (mode === 'immersive-ar') throw new Error('AR probe unavailable') + return true + }), + requestSession: vi.fn() + } + })).resolves.toMatchObject({ + status: 'available', + modes: { vr: true, ar: false }, + entryMode: 'immersive-vr' + }) + }) + it('reports insecure and unsupported browsers with an actionable fallback', async () => { await expect(detectImmersiveCapability({ secureContext: false @@ -56,6 +107,59 @@ describe('immersive WebXR capability', () => { ) }) + it('requests immersive AR explicitly and includes DOM overlay only when provided', async () => { + const session = {} as XRSession + const requestSession = vi.fn(async () => session) + const root = {} as Element + await requestImmersiveSession({ + secureContext: true, + xr: { isSessionSupported: vi.fn(async () => true), requestSession } + }, 'immersive-ar', root) + expect(requestSession).toHaveBeenCalledWith('immersive-ar', { + requiredFeatures: ['local-floor'], + optionalFeatures: ['bounded-floor', 'hand-tracking', 'layers', 'dom-overlay'], + domOverlay: { root } + }) + }) + + it('reports the honest passthrough capability and blend-mode matrix', () => { + expect(resolvePassthroughAvailability({ + arSupported: false, + vrSupported: true, + mode: 'immersive-vr', + environmentBlendMode: 'opaque' + })).toMatchObject({ supported: false, active: false }) + expect(resolvePassthroughAvailability({ + arSupported: true, + vrSupported: true, + mode: 'immersive-vr', + environmentBlendMode: 'opaque' + })).toMatchObject({ supported: true, active: false }) + expect(resolvePassthroughAvailability({ + arSupported: true, + vrSupported: true, + mode: 'immersive-ar', + environmentBlendMode: 'alpha-blend' + })).toMatchObject({ supported: true, active: true, contrast: 'dither' }) + expect(resolvePassthroughAvailability({ + arSupported: true, + vrSupported: true, + mode: 'immersive-ar', + environmentBlendMode: 'additive' + })).toMatchObject({ supported: true, active: true, contrast: 'additive-shape' }) + expect(resolvePassthroughAvailability({ + arSupported: true, + vrSupported: true, + mode: 'immersive-ar', + environmentBlendMode: 'opaque' + })).toMatchObject({ + supported: true, + active: false, + contrast: 'opaque', + disabledReason: null + }) + }) + it('refuses session creation outside a secure context', async () => { const requestSession = vi.fn() await expect(requestImmersiveSession({