From fbf8cc7afa8664972d0748cb17e9857b65beaf26 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Mon, 14 Sep 2026 16:25:46 +0300 Subject: [PATCH 1/5] test(e2e): Add a node-flue end-to-end application MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The node-integration-test suite drives Flue from a hand-written scenario that calls `__flueBindAgentModule` itself, standing in for what `@flue/vite` does at build time. It cannot show whether a scaffolded app works. This app is what `flue init` produces — plain vite, `'use agent'`, `createAgentRouter` — built and served the way a user runs it, against a real provider. Covers: AI spans, errors captured as issues, a manual span nesting inside a tool, an orchestrion-instrumented `dataloader` span landing in the agent's trace, and both dev and prod. Also asserts the provider's HTTP call nests inside `chat`, which nothing else covers. The loader is called from inside a tool rather than a route, so its span shares the agent's trace instead of sitting in one of its own. No build externals are needed, unlike node-eve: a Flue node build leaves dependencies as bare specifiers, so `dataloader` stays a real module for the transform to hook. The `@flue/*` versions are pinned because the internal registry proxy 403s on releases it has not scanned, and a caret range drifts onto them; the `(latest)` variant is where new versions get exercised. Co-Authored-By: Claude Opus 5 --- .../test-applications/node-flue/.gitignore | 7 ++ .../node-flue/flue.config.ts | 5 ++ .../test-applications/node-flue/package.json | 59 ++++++++++++++ .../node-flue/playwright.config.mjs | 23 ++++++ .../node-flue/sentry-init.ts | 17 ++++ .../node-flue/src/agents/hello.ts | 47 +++++++++++ .../test-applications/node-flue/src/app.ts | 10 +++ .../node-flue/src/loaders.ts | 5 ++ .../node-flue/start-event-proxy.mjs | 6 ++ .../node-flue/tests/dataloader.test.ts | 46 +++++++++++ .../node-flue/tests/errors.test.ts | 21 +++++ .../node-flue/tests/flue.test.ts | 77 +++++++++++++++++++ .../node-flue/tests/utils.ts | 21 +++++ .../test-applications/node-flue/tsconfig.json | 14 ++++ .../node-flue/vite.config.ts | 9 +++ 15 files changed, 367 insertions(+) create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/.gitignore create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/flue.config.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/package.json create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/playwright.config.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/sentry-init.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/src/app.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/start-event-proxy.mjs create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/tsconfig.json create mode 100644 dev-packages/e2e-tests/test-applications/node-flue/vite.config.ts diff --git a/dev-packages/e2e-tests/test-applications/node-flue/.gitignore b/dev-packages/e2e-tests/test-applications/node-flue/.gitignore new file mode 100644 index 000000000000..2685f6ec088a --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/.gitignore @@ -0,0 +1,7 @@ +node_modules +dist +.flue +*.tsbuildinfo +results.junit.xml +test-results +playwright-report diff --git a/dev-packages/e2e-tests/test-applications/node-flue/flue.config.ts b/dev-packages/e2e-tests/test-applications/node-flue/flue.config.ts new file mode 100644 index 000000000000..a31c8523a9a7 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/flue.config.ts @@ -0,0 +1,5 @@ +import { defineConfig } from '@flue/runtime/config'; + +export default defineConfig({ + target: 'node', +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/package.json b/dev-packages/e2e-tests/test-applications/node-flue/package.json new file mode 100644 index 000000000000..7d38b0bd4210 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/package.json @@ -0,0 +1,59 @@ +{ + "name": "node-flue", + "version": "0.0.0", + "private": true, + "type": "module", + "scripts": { + "dev": "vite dev --port 3030", + "build": "vite build", + "start": "PORT=3030 node dist/server.mjs", + "dev:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm dev", + "start:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm start", + "clean": "npx rimraf node_modules dist pnpm-lock.yaml", + "test:build": "pnpm install && pnpm build", + "test:build-orchestrion": "USE_ORCHESTRION=1 pnpm test:build", + "test:build-latest": "pnpm install && pnpm add @flue/runtime@latest @flue/vite@latest @flue/cli@latest && pnpm build", + "test:assert": "pnpm test:prod && pnpm test:dev", + "test:assert-orchestrion": "USE_ORCHESTRION=1 pnpm test:assert", + "test:prod": "OPENROUTER_API_KEY=$E2E_OPENROUTER_API_KEY TEST_ENV=production playwright test", + "test:dev": "OPENROUTER_API_KEY=$E2E_OPENROUTER_API_KEY TEST_ENV=development playwright test" + }, + "dependencies": { + "@flue/runtime": "2.0.5", + "@sentry/node": "file:../../packed/sentry-node-packed.tgz", + "dataloader": "^2.2.3", + "hono": "^4.7.0", + "valibot": "^1.5.0" + }, + "devDependencies": { + "@flue/cli": "2.0.5", + "@flue/vite": "2.0.5", + "@playwright/test": "~1.56.0", + "@sentry-internal/test-utils": "link:../../../test-utils", + "@sentry/core": "file:../../packed/sentry-core-packed.tgz", + "@types/node": "24.x", + "typescript": "~5.9.0", + "vite": "^8.0.14" + }, + "engines": { + "node": "24.x" + }, + "volta": { + "node": "24.15.0", + "extends": "../../package.json" + }, + "sentryTest": { + "optional": true, + "optionalVariants": [ + { + "build-command": "pnpm test:build-latest", + "label": "node-flue (latest)" + }, + { + "build-command": "pnpm test:build-orchestrion", + "assert-command": "pnpm test:assert-orchestrion", + "label": "node-flue (orchestrion)" + } + ] + } +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-flue/playwright.config.mjs new file mode 100644 index 000000000000..15da94e25274 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/playwright.config.mjs @@ -0,0 +1,23 @@ +import { getPlaywrightConfig } from '@sentry-internal/test-utils'; + +const testEnv = process.env.TEST_ENV; +const useOrchestrion = process.env.USE_ORCHESTRION === '1'; + +if (!testEnv) { + throw new Error('No test env defined'); +} + +let startCommand = testEnv === 'development' ? 'pnpm dev' : 'pnpm start'; + +if (useOrchestrion) { + startCommand = `${startCommand}:orchestrion`; +} + +const config = getPlaywrightConfig( + { startCommand }, + // Each agent turn is a real OpenRouter tool-calling round trip (two model calls) followed by a + // span flush, which does not fit the default 30s timeout when the provider is slow. + { timeout: 90_000 }, +); + +export default config; diff --git a/dev-packages/e2e-tests/test-applications/node-flue/sentry-init.ts b/dev-packages/e2e-tests/test-applications/node-flue/sentry-init.ts new file mode 100644 index 000000000000..c17c814a3663 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/sentry-init.ts @@ -0,0 +1,17 @@ +import { instrument } from '@flue/runtime'; +import * as Sentry from '@sentry/node'; + +// Imported for its side effects as the first line of `src/app.ts`, which is how a Flue app is +// expected to set Sentry up: there is no framework-owned instrumentation hook to auto-discover. +Sentry.init({ + environment: 'qa', + dsn: process.env.E2E_TEST_DSN, + tunnel: 'http://localhost:3031/', // proxy server + tracesSampleRate: 1.0, + // Not a default integration. It only produces spans in the "orchestrion" test variant, where the + // server starts with `NODE_OPTIONS=--import=@sentry/node/import` so the module transform is + // registered before `dataloader` is loaded. + integrations: [Sentry.dataloaderIntegration()], +}); + +instrument(Sentry.createFlueInstrumentation()); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts new file mode 100644 index 000000000000..093616fa1901 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts @@ -0,0 +1,47 @@ +'use agent'; +import { useModel, useTool } from '@flue/runtime'; +import * as Sentry from '@sentry/node'; +import * as v from 'valibot'; +import { itemLoader } from '../loaders.ts'; + +// The `'use agent'` directive is how `@flue/vite` finds this module and binds an identity to it at +// build time. That binding is the part a hand-written scenario cannot reproduce, so it is the main +// reason this app exists alongside the node-integration-test suite. +export function Hello() { + useModel('openrouter/anthropic/claude-haiku-4.5'); + + useTool({ + name: 'get_weather', + description: 'Get the current weather for a city.', + input: v.object({ city: v.string() }), + // Wrapped in a manual span: Flue runs the tool while the SDK's `execute_tool` span is active, + // so this should nest directly under it rather than landing beside it. + run: ({ city }) => + Sentry.startSpan({ name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': city } }, () => { + return `It is 21 degrees and sunny in ${city}.`; + }), + }); + + // Called from inside a tool on purpose: the dataloader span then lands under `execute_tool` in + // the agent's trace, which is what "captured alongside the AI spans" has to mean. + useTool({ + name: 'count_items', + description: 'Count items by loading them. Call this when the user asks to count items.', + input: v.object({}), + run: async () => { + const doubled = await Promise.all([itemLoader.load(1), itemLoader.load(2), itemLoader.load(3)]); + return `Loaded ${doubled.length} items: ${doubled.join(', ')}.`; + }, + }); + + useTool({ + name: 'fail_now', + description: 'Always throws an error. Call this when the user asks to trigger a failure.', + input: v.object({}), + run: () => { + throw new Error('Intentional flue tool failure'); + }, + }); + + return 'You are a helpful assistant. Use get_weather when asked about weather, and fail_now when asked to fail.'; +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/app.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/app.ts new file mode 100644 index 000000000000..b54a3357e4ca --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/src/app.ts @@ -0,0 +1,10 @@ +import '../sentry-init.ts'; +import { createAgentRouter } from '@flue/runtime/routing'; +import { Hono } from 'hono'; +import { Hello } from './agents/hello.ts'; + +const app = new Hono(); + +app.route('/agents/hello', createAgentRouter(Hello)); + +export default app; diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts new file mode 100644 index 000000000000..32bbb1ee4515 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts @@ -0,0 +1,5 @@ +import DataLoader from 'dataloader'; + +// Exercised through a plain route rather than a tool, so the orchestrion assertion does not depend +// on a model call deciding to invoke it. +export const itemLoader = new DataLoader(async keys => keys.map(key => key * 2)); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/start-event-proxy.mjs b/dev-packages/e2e-tests/test-applications/node-flue/start-event-proxy.mjs new file mode 100644 index 000000000000..a452f0dd18d3 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/start-event-proxy.mjs @@ -0,0 +1,6 @@ +import { startEventProxyServer } from '@sentry-internal/test-utils'; + +startEventProxyServer({ + port: 3031, + proxyServerName: 'node-flue', +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts new file mode 100644 index 000000000000..67953afa19db --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts @@ -0,0 +1,46 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; +import { runAgentTurn } from './utils'; + +const APP = 'node-flue'; +const useOrchestrion = process.env.USE_ORCHESTRION === '1'; + +const isDataloaderSpan = (span: { attributes?: Record }): boolean => + span.attributes?.['sentry.origin']?.value === 'auto.db.dataloader'; + +/** + * `dataloaderIntegration` relies on orchestrion, a module transform, so it only emits spans when the + * server starts with `NODE_OPTIONS=--import=@sentry/node/import` (the `node-flue (orchestrion)` + * variant). The integration installs and subscribes either way, so the absence of a span is the only + * thing that distinguishes the two — hence `test.fail(!useOrchestrion)`. + * + * Flue needs no build configuration for this: a Flue node build leaves dependencies as bare + * specifiers, so `dataloader` stays a real module the transform can hook. If Flue ever switches to + * a bundled server output, this test is what catches it. + * + * The loader is called from inside a tool so its span lands in the agent's trace, beside the AI + * spans, rather than in a trace of its own. + */ +test('captures orchestrion-instrumented dataloader spans in the same trace as the AI spans', async ({ baseURL }) => { + test.fail(!useOrchestrion, 'orchestrion module instrumentation needs NODE_OPTIONS=--import=@sentry/node/import'); + + // With orchestrion, wait for the dataloader span itself. Without it that span never arrives, so + // anchor on the always-present tool span and let the assertion below fail fast rather than time + // the test out. + const spansPromise = collectStreamedSpans(APP, spansOfTrace => + useOrchestrion + ? spansOfTrace.some(isDataloaderSpan) + : spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool'), + ); + + await runAgentTurn(baseURL!, 'dataloader-conversation', 'Please call count_items to count the items.'); + + const spans = await spansPromise; + const executeTool = spans.find(span => span.attributes?.['gen_ai.tool.name']?.value === 'count_items'); + const dataloaderSpan = spans.find(isDataloaderSpan); + + expect(dataloaderSpan?.attributes?.['sentry.origin']?.value).toBe('auto.db.dataloader'); + // Same trace as the AI spans, and underneath the tool that triggered it. + expect(dataloaderSpan?.trace_id).toBe(executeTool?.trace_id); + expect(dataloaderSpan?.parent_span_id).toBe(executeTool?.span_id); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts new file mode 100644 index 000000000000..dccbca210c68 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts @@ -0,0 +1,21 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils'; +import { runAgentTurn } from './utils'; + +const APP = 'node-flue'; + +test('captures an error thrown inside a Flue tool and marks its span errored', async ({ baseURL }) => { + const errorPromise = waitForError(APP, event => event.exception?.values?.[0]?.value === 'Intentional flue tool failure'); + const spansPromise = collectStreamedSpans(APP, spansOfTrace => + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool'), + ); + + await runAgentTurn(baseURL!, 'failure-conversation', 'Please call fail_now to trigger a failure.'); + + const error = await errorPromise; + expect(error.exception?.values?.[0]?.value).toBe('Intentional flue tool failure'); + + const spans = await spansPromise; + const executeTool = spans.find(span => span.attributes?.['gen_ai.tool.name']?.value === 'fail_now'); + expect(executeTool?.status).toBe('error'); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts new file mode 100644 index 000000000000..be32a4c90b52 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts @@ -0,0 +1,77 @@ +import { expect, test } from '@playwright/test'; +import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; +import { runAgentTurn } from './utils'; + +const APP = 'node-flue'; + +const hasOps = (ops: string[]) => (spansOfTrace: { attributes?: Record }[]) => + ops.every(op => spansOfTrace.some(span => getSpanOp(span) === op)); + +test('captures the invoke_agent / chat / execute_tool hierarchy for a Flue turn', async ({ baseURL }) => { + // The trace flushes across several envelopes, so accumulate it rather than asserting on one. + const spansPromise = collectStreamedSpans(APP, hasOps(['gen_ai.invoke_agent', 'gen_ai.chat', 'gen_ai.execute_tool'])); + + await runAgentTurn(baseURL!, 'weather-conversation', 'What is the weather in Paris?'); + + const spans = await spansPromise; + const invokeAgent = spans.find(span => getSpanOp(span) === 'gen_ai.invoke_agent'); + const chat = spans.find(span => getSpanOp(span) === 'gen_ai.chat'); + const executeTool = spans.find(span => getSpanOp(span) === 'gen_ai.execute_tool'); + + expect(invokeAgent?.attributes?.['sentry.origin']?.value).toBe('auto.ai.flue'); + expect(invokeAgent?.attributes?.['gen_ai.operation.name']?.value).toBe('invoke_agent'); + expect(invokeAgent?.attributes?.['gen_ai.agent.name']?.value).toBe('Hello'); + + expect(chat?.attributes?.['sentry.origin']?.value).toBe('auto.ai.flue'); + expect(chat?.attributes?.['gen_ai.provider.name']?.value).toBe('openrouter'); + expect(typeof chat?.attributes?.['gen_ai.usage.input_tokens']?.value).toBe('number'); + expect(typeof chat?.attributes?.['gen_ai.usage.output_tokens']?.value).toBe('number'); + // Flue computes cost itself; no provider SDK reports it. + expect(typeof chat?.attributes?.['gen_ai.cost.total_tokens']?.value).toBe('number'); + + expect(executeTool?.attributes?.['gen_ai.tool.name']?.value).toBe('get_weather'); + + // Tool and chat spans are siblings under the agent invocation, matching how Flue's own + // OpenTelemetry adapter projects them. + expect(chat?.parent_span_id).toBe(invokeAgent?.span_id); + expect(executeTool?.parent_span_id).toBe(invokeAgent?.span_id); +}); + +test('nests a manual span raised inside a tool under that tool span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool') && + spansOfTrace.some(span => span.name === 'resolve-weather'), + ); + + await runAgentTurn(baseURL!, 'manual-span-conversation', 'What is the weather in Berlin?'); + + const spans = await spansPromise; + const executeTool = spans.find(span => getSpanOp(span) === 'gen_ai.execute_tool'); + const manualSpan = spans.find(span => span.name === 'resolve-weather'); + + expect(manualSpan?.attributes?.['weather.source']?.value).toBe('static-table'); + expect(manualSpan?.trace_id).toBe(executeTool?.trace_id); + expect(manualSpan?.parent_span_id).toBe(executeTool?.span_id); +}); + +// Flue's `model` operation is wrapped so the turn span is active for it, which is what puts the +// provider's HTTP call inside `chat` rather than beside it. +test('nests the provider HTTP call inside the chat span', async ({ baseURL }) => { + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.chat') && + spansOfTrace.some(span => getSpanOp(span) === 'http.client'), + ); + + await runAgentTurn(baseURL!, 'provider-http-conversation', 'Say hello.'); + + const spans = await spansPromise; + const chat = spans.find(span => getSpanOp(span) === 'gen_ai.chat'); + const providerCall = spans.find(span => getSpanOp(span) === 'http.client'); + + expect(providerCall?.trace_id).toBe(chat?.trace_id); + expect(providerCall?.parent_span_id).toBe(chat?.span_id); +}); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts new file mode 100644 index 000000000000..ca2fc1fc58b5 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts @@ -0,0 +1,21 @@ +import { expect } from '@playwright/test'; + +/** + * Start one agent turn over Flue's agent router. + * + * `createAgentRouter` mounts `POST /:id`, which accepts the prompt and returns `202` with a + * `streamUrl` — the turn itself runs afterwards. So this only starts the work; callers wait on the + * spans they expect, which `collectStreamedSpans` accumulates across envelopes. + * + * The conversation id is ours to choose: it is the `:id` path segment. + */ +export async function runAgentTurn(baseURL: string, conversationId: string, message: string): Promise { + const res = await fetch(`${baseURL}/agents/hello/${conversationId}`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ kind: 'user', body: message }), + }); + + expect(res.status).toBe(202); + await res.text(); +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tsconfig.json b/dev-packages/e2e-tests/test-applications/node-flue/tsconfig.json new file mode 100644 index 000000000000..2cb98b13c9f8 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "esnext", + "moduleResolution": "bundler", + "types": ["node"], + "strict": true, + "allowImportingTsExtensions": true, + "esModuleInterop": true, + "skipLibCheck": true, + "noEmit": true + }, + "include": ["src/**/*.ts", "sentry-init.ts", "flue.config.ts", "vite.config.ts"] +} diff --git a/dev-packages/e2e-tests/test-applications/node-flue/vite.config.ts b/dev-packages/e2e-tests/test-applications/node-flue/vite.config.ts new file mode 100644 index 000000000000..8193cc038588 --- /dev/null +++ b/dev-packages/e2e-tests/test-applications/node-flue/vite.config.ts @@ -0,0 +1,9 @@ +import { flue } from '@flue/vite'; +import { defineConfig } from 'vite'; + +// Unmodified from what `flue init` scaffolds. In particular there is no externals config: a Flue +// node build leaves dependencies as bare specifiers, so orchestrion's module transform still sees +// them as real modules. (eve needs `externalDependencies` because it emits a bundled server.) +export default defineConfig({ + plugins: [flue()], +}); From 148b9e7be6ff65ec412ffcd3863c91405e8e5360 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Mon, 14 Sep 2026 17:37:45 +0300 Subject: [PATCH 2/5] style(e2e): Format the node-flue app Two lines over the width limit; I formatted the server-utils sources but not the test application. Co-Authored-By: Claude Opus 5 --- .../test-applications/node-flue/src/agents/hello.ts | 9 ++++++--- .../test-applications/node-flue/tests/errors.test.ts | 5 ++++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts index 093616fa1901..35203267eefd 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts @@ -17,9 +17,12 @@ export function Hello() { // Wrapped in a manual span: Flue runs the tool while the SDK's `execute_tool` span is active, // so this should nest directly under it rather than landing beside it. run: ({ city }) => - Sentry.startSpan({ name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': city } }, () => { - return `It is 21 degrees and sunny in ${city}.`; - }), + Sentry.startSpan( + { name: 'resolve-weather', attributes: { 'weather.source': 'static-table', 'weather.city': city } }, + () => { + return `It is 21 degrees and sunny in ${city}.`; + }, + ), }); // Called from inside a tool on purpose: the dataloader span then lands under `execute_tool` in diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts index dccbca210c68..11652d339618 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts @@ -5,7 +5,10 @@ import { runAgentTurn } from './utils'; const APP = 'node-flue'; test('captures an error thrown inside a Flue tool and marks its span errored', async ({ baseURL }) => { - const errorPromise = waitForError(APP, event => event.exception?.values?.[0]?.value === 'Intentional flue tool failure'); + const errorPromise = waitForError( + APP, + event => event.exception?.values?.[0]?.value === 'Intentional flue tool failure', + ); const spansPromise = collectStreamedSpans(APP, spansOfTrace => spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool'), ); From 707b1f545057caaf7a72656f908405ee9384cbc8 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Mon, 14 Sep 2026 18:23:12 +0300 Subject: [PATCH 3/5] test(e2e): Wait for the Flue turn to settle before asserting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `runAgentTurn` returned on the `202` and dropped the `streamUrl`, so a turn kept running while the next test started waiting for spans — a leftover trace could satisfy the wrong assertion. It now reads the conversation back until it reports a settlement. Scoping the waits by `gen_ai.conversation.id` would not have worked: Flue generates that id (`conv_01M2G81…`), so it is not the path segment the test chose and the test cannot know it up front. Also names `count_items` in the agent instructions — an earlier edit missed, so the dataloader test was relying on the tool description alone — and drops the `loaders.ts` comment describing the route-based setup that no longer exists. Co-Authored-By: Claude Opus 5 --- .../node-flue/src/agents/hello.ts | 2 +- .../node-flue/src/loaders.ts | 2 -- .../node-flue/tests/utils.ts | 28 ++++++++++++++----- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts index 35203267eefd..31cee2ba0499 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts @@ -46,5 +46,5 @@ export function Hello() { }, }); - return 'You are a helpful assistant. Use get_weather when asked about weather, and fail_now when asked to fail.'; + return 'You are a helpful assistant. Use get_weather when asked about weather, count_items when asked to count items, and fail_now when asked to fail.'; } diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts index 32bbb1ee4515..df820fc45728 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts @@ -1,5 +1,3 @@ import DataLoader from 'dataloader'; -// Exercised through a plain route rather than a tool, so the orchestrion assertion does not depend -// on a model call deciding to invoke it. export const itemLoader = new DataLoader(async keys => keys.map(key => key * 2)); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts index ca2fc1fc58b5..e214f4abcea6 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts @@ -1,21 +1,35 @@ import { expect } from '@playwright/test'; /** - * Start one agent turn over Flue's agent router. + * Run one agent turn over Flue's agent router and wait for it to settle. * - * `createAgentRouter` mounts `POST /:id`, which accepts the prompt and returns `202` with a - * `streamUrl` — the turn itself runs afterwards. So this only starts the work; callers wait on the - * spans they expect, which `collectStreamedSpans` accumulates across envelopes. + * `POST /:id` only admits the work — it returns `202` with a `streamUrl` and the turn runs after. + * Returning there would let one test's turn still be emitting spans while the next one waits for + * spans of its own, so a leftover trace could satisfy the wrong assertion. Reading the conversation + * back until it reports a settlement keeps each test to its own turn. * - * The conversation id is ours to choose: it is the `:id` path segment. + * The conversation id is ours to choose: it is the `:id` path segment. It is not the + * `gen_ai.conversation.id` attribute, which Flue generates. */ export async function runAgentTurn(baseURL: string, conversationId: string, message: string): Promise { - const res = await fetch(`${baseURL}/agents/hello/${conversationId}`, { + const url = `${baseURL}/agents/hello/${conversationId}`; + + const res = await fetch(url, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ kind: 'user', body: message }), }); - expect(res.status).toBe(202); await res.text(); + + const deadline = Date.now() + 60_000; + while (Date.now() < deadline) { + const conversation = (await (await fetch(url)).json()) as { settlements?: unknown[] }; + if (conversation.settlements?.length) { + return; + } + await new Promise(resolve => setTimeout(resolve, 250)); + } + + throw new Error(`Flue turn for "${conversationId}" did not settle within 60s`); } From 30d7d91c05f1cf3078efd8315381a7229d0dfc36 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Tue, 15 Sep 2026 12:01:58 +0300 Subject: [PATCH 4/5] test(e2e): Address review on the node-flue app - Drop the explicit `dataloaderIntegration()`; it is in `getTracingIntegrations()` now, so Node registers it by default when spans are enabled. - Always run under orchestrion rather than keeping it as a variant, since that is how the SDK is meant to be set up. Removes the `*:orchestrion` scripts, the `USE_ORCHESTRION` plumbing and the `test.fail()` branch in the dataloader test, which now simply asserts the span lands under `execute_tool`. - Drop the node-eve reference from `vite.config.ts`. Co-Authored-By: Claude Opus 5 --- .../test-applications/node-flue/package.json | 13 ++------- .../node-flue/playwright.config.mjs | 13 +++------ .../node-flue/sentry-init.ts | 8 ++---- .../node-flue/tests/dataloader.test.ts | 27 +++++-------------- .../node-flue/vite.config.ts | 6 ++--- 5 files changed, 16 insertions(+), 51 deletions(-) diff --git a/dev-packages/e2e-tests/test-applications/node-flue/package.json b/dev-packages/e2e-tests/test-applications/node-flue/package.json index 7d38b0bd4210..7f6afaeff6b8 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/package.json +++ b/dev-packages/e2e-tests/test-applications/node-flue/package.json @@ -4,17 +4,13 @@ "private": true, "type": "module", "scripts": { - "dev": "vite dev --port 3030", + "dev": "NODE_OPTIONS='--import=@sentry/node/import' vite dev --port 3030", "build": "vite build", - "start": "PORT=3030 node dist/server.mjs", - "dev:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm dev", - "start:orchestrion": "NODE_OPTIONS='--import=@sentry/node/import' pnpm start", + "start": "NODE_OPTIONS='--import=@sentry/node/import' PORT=3030 node dist/server.mjs", "clean": "npx rimraf node_modules dist pnpm-lock.yaml", "test:build": "pnpm install && pnpm build", - "test:build-orchestrion": "USE_ORCHESTRION=1 pnpm test:build", "test:build-latest": "pnpm install && pnpm add @flue/runtime@latest @flue/vite@latest @flue/cli@latest && pnpm build", "test:assert": "pnpm test:prod && pnpm test:dev", - "test:assert-orchestrion": "USE_ORCHESTRION=1 pnpm test:assert", "test:prod": "OPENROUTER_API_KEY=$E2E_OPENROUTER_API_KEY TEST_ENV=production playwright test", "test:dev": "OPENROUTER_API_KEY=$E2E_OPENROUTER_API_KEY TEST_ENV=development playwright test" }, @@ -48,11 +44,6 @@ { "build-command": "pnpm test:build-latest", "label": "node-flue (latest)" - }, - { - "build-command": "pnpm test:build-orchestrion", - "assert-command": "pnpm test:assert-orchestrion", - "label": "node-flue (orchestrion)" } ] } diff --git a/dev-packages/e2e-tests/test-applications/node-flue/playwright.config.mjs b/dev-packages/e2e-tests/test-applications/node-flue/playwright.config.mjs index 15da94e25274..1bbde34c9fce 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/playwright.config.mjs +++ b/dev-packages/e2e-tests/test-applications/node-flue/playwright.config.mjs @@ -1,22 +1,15 @@ import { getPlaywrightConfig } from '@sentry-internal/test-utils'; const testEnv = process.env.TEST_ENV; -const useOrchestrion = process.env.USE_ORCHESTRION === '1'; if (!testEnv) { throw new Error('No test env defined'); } -let startCommand = testEnv === 'development' ? 'pnpm dev' : 'pnpm start'; - -if (useOrchestrion) { - startCommand = `${startCommand}:orchestrion`; -} - const config = getPlaywrightConfig( - { startCommand }, - // Each agent turn is a real OpenRouter tool-calling round trip (two model calls) followed by a - // span flush, which does not fit the default 30s timeout when the provider is slow. + { startCommand: testEnv === 'development' ? 'pnpm dev' : 'pnpm start' }, + // Each test drives a real OpenRouter tool-calling turn and then waits for the spans to flush, + // which does not fit the default 30s timeout when the provider is slow. { timeout: 90_000 }, ); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/sentry-init.ts b/dev-packages/e2e-tests/test-applications/node-flue/sentry-init.ts index c17c814a3663..199d1bb11fcd 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/sentry-init.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/sentry-init.ts @@ -1,17 +1,13 @@ import { instrument } from '@flue/runtime'; import * as Sentry from '@sentry/node'; -// Imported for its side effects as the first line of `src/app.ts`, which is how a Flue app is -// expected to set Sentry up: there is no framework-owned instrumentation hook to auto-discover. +// Imported for its side effects as the first line of `src/app.ts`, which is how a Flue app sets +// Sentry up: there is no framework-owned instrumentation hook to auto-discover. Sentry.init({ environment: 'qa', dsn: process.env.E2E_TEST_DSN, tunnel: 'http://localhost:3031/', // proxy server tracesSampleRate: 1.0, - // Not a default integration. It only produces spans in the "orchestrion" test variant, where the - // server starts with `NODE_OPTIONS=--import=@sentry/node/import` so the module transform is - // registered before `dataloader` is loaded. - integrations: [Sentry.dataloaderIntegration()], }); instrument(Sentry.createFlueInstrumentation()); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts index 67953afa19db..26e86a542e6c 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts @@ -3,35 +3,21 @@ import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; import { runAgentTurn } from './utils'; const APP = 'node-flue'; -const useOrchestrion = process.env.USE_ORCHESTRION === '1'; const isDataloaderSpan = (span: { attributes?: Record }): boolean => span.attributes?.['sentry.origin']?.value === 'auto.db.dataloader'; /** - * `dataloaderIntegration` relies on orchestrion, a module transform, so it only emits spans when the - * server starts with `NODE_OPTIONS=--import=@sentry/node/import` (the `node-flue (orchestrion)` - * variant). The integration installs and subscribes either way, so the absence of a span is the only - * thing that distinguishes the two — hence `test.fail(!useOrchestrion)`. - * - * Flue needs no build configuration for this: a Flue node build leaves dependencies as bare - * specifiers, so `dataloader` stays a real module the transform can hook. If Flue ever switches to - * a bundled server output, this test is what catches it. + * `dataloader` is instrumented through orchestrion, a module transform, so it only produces spans + * with the loader registered at process start. A Flue node build needs no externals config for + * that: dependencies stay bare specifiers, so `dataloader` is still a real module to hook. If Flue + * ever switches to a bundled server output, this is what catches it. * * The loader is called from inside a tool so its span lands in the agent's trace, beside the AI * spans, rather than in a trace of its own. */ test('captures orchestrion-instrumented dataloader spans in the same trace as the AI spans', async ({ baseURL }) => { - test.fail(!useOrchestrion, 'orchestrion module instrumentation needs NODE_OPTIONS=--import=@sentry/node/import'); - - // With orchestrion, wait for the dataloader span itself. Without it that span never arrives, so - // anchor on the always-present tool span and let the assertion below fail fast rather than time - // the test out. - const spansPromise = collectStreamedSpans(APP, spansOfTrace => - useOrchestrion - ? spansOfTrace.some(isDataloaderSpan) - : spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool'), - ); + const spansPromise = collectStreamedSpans(APP, spansOfTrace => spansOfTrace.some(isDataloaderSpan)); await runAgentTurn(baseURL!, 'dataloader-conversation', 'Please call count_items to count the items.'); @@ -39,8 +25,7 @@ test('captures orchestrion-instrumented dataloader spans in the same trace as th const executeTool = spans.find(span => span.attributes?.['gen_ai.tool.name']?.value === 'count_items'); const dataloaderSpan = spans.find(isDataloaderSpan); - expect(dataloaderSpan?.attributes?.['sentry.origin']?.value).toBe('auto.db.dataloader'); - // Same trace as the AI spans, and underneath the tool that triggered it. + expect(getSpanOp(dataloaderSpan!)).toBe('cache.get'); expect(dataloaderSpan?.trace_id).toBe(executeTool?.trace_id); expect(dataloaderSpan?.parent_span_id).toBe(executeTool?.span_id); }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/vite.config.ts b/dev-packages/e2e-tests/test-applications/node-flue/vite.config.ts index 8193cc038588..2f495966e204 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/vite.config.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/vite.config.ts @@ -1,9 +1,9 @@ import { flue } from '@flue/vite'; import { defineConfig } from 'vite'; -// Unmodified from what `flue init` scaffolds. In particular there is no externals config: a Flue -// node build leaves dependencies as bare specifiers, so orchestrion's module transform still sees -// them as real modules. (eve needs `externalDependencies` because it emits a bundled server.) +// Unmodified from what `flue init` scaffolds. No externals config is needed: a Flue node build +// already leaves dependencies as bare specifiers, so orchestrion's module transform still sees them +// as real modules. export default defineConfig({ plugins: [flue()], }); From 9abab729dbe18f28bcc683c4e145ef1e76e30be1 Mon Sep 17 00:00:00 2001 From: RulaKhaled Date: Tue, 15 Sep 2026 14:42:07 +0300 Subject: [PATCH 5/5] test(e2e): Make the node-flue suite deterministic Three things made it flaky on a Playwright retry, which is why the dev run went red then green: - `DataLoader` was a module-level singleton, so it cached keys 1-3 and a second `count_items` call skipped the batch function and emitted no span. Constructed per execution now, matching node-eve. - Conversation ids were fixed strings, so `runAgentTurn` saw a settlement from an earlier run and returned before the new turn finished. Each turn gets its own. - The waits matched any agent turn, so a leftover trace could satisfy the wrong test. Each now anchors on its own tool via `gen_ai.tool.name`. The dataloader test asserts a shared trace rather than the exact parent: the model may call the tool more than once, and the span that ran the loader is not reliably the one found by name. Co-Authored-By: Claude Opus 5 --- .../node-flue/src/agents/hello.ts | 7 +++-- .../node-flue/src/loaders.ts | 3 --- .../node-flue/tests/dataloader.test.ts | 19 ++++++++----- .../node-flue/tests/errors.test.ts | 8 +++--- .../node-flue/tests/flue.test.ts | 27 ++++++++++++------- .../node-flue/tests/utils.ts | 11 ++++++++ 6 files changed, 51 insertions(+), 24 deletions(-) delete mode 100644 dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts index 31cee2ba0499..baf29050d836 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/src/agents/hello.ts @@ -2,7 +2,7 @@ import { useModel, useTool } from '@flue/runtime'; import * as Sentry from '@sentry/node'; import * as v from 'valibot'; -import { itemLoader } from '../loaders.ts'; +import DataLoader from 'dataloader'; // The `'use agent'` directive is how `@flue/vite` finds this module and binds an identity to it at // build time. That binding is the part a hand-written scenario cannot reproduce, so it is the main @@ -31,8 +31,11 @@ export function Hello() { name: 'count_items', description: 'Count items by loading them. Call this when the user asks to count items.', input: v.object({}), + // Constructed per execution, like node-eve does: a module-level loader caches its keys, so a + // second call would skip the batch function and emit no span. run: async () => { - const doubled = await Promise.all([itemLoader.load(1), itemLoader.load(2), itemLoader.load(3)]); + const loader = new DataLoader(async keys => keys.map(key => key * 2)); + const doubled = await Promise.all([loader.load(1), loader.load(2), loader.load(3)]); return `Loaded ${doubled.length} items: ${doubled.join(', ')}.`; }, }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts b/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts deleted file mode 100644 index df820fc45728..000000000000 --- a/dev-packages/e2e-tests/test-applications/node-flue/src/loaders.ts +++ /dev/null @@ -1,3 +0,0 @@ -import DataLoader from 'dataloader'; - -export const itemLoader = new DataLoader(async keys => keys.map(key => key * 2)); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts index 26e86a542e6c..a4fed849c3b2 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/dataloader.test.ts @@ -1,6 +1,6 @@ import { expect, test } from '@playwright/test'; import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; -import { runAgentTurn } from './utils'; +import { newConversationId, runAgentTurn } from './utils'; const APP = 'node-flue'; @@ -17,15 +17,22 @@ const isDataloaderSpan = (span: { attributes?: Record { - const spansPromise = collectStreamedSpans(APP, spansOfTrace => spansOfTrace.some(isDataloaderSpan)); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + spansOfTrace.some(span => span.attributes?.['gen_ai.tool.name']?.value === 'count_items') && + spansOfTrace.some(isDataloaderSpan), + ); - await runAgentTurn(baseURL!, 'dataloader-conversation', 'Please call count_items to count the items.'); + await runAgentTurn(baseURL!, newConversationId('dataloader'), 'Please call count_items to count the items.'); const spans = await spansPromise; - const executeTool = spans.find(span => span.attributes?.['gen_ai.tool.name']?.value === 'count_items'); const dataloaderSpan = spans.find(isDataloaderSpan); + const toolSpan = spans.find(span => span.attributes?.['gen_ai.tool.name']?.value === 'count_items'); + // Sharing the trace is the point: the orchestrion span is captured alongside the AI spans rather + // than in a trace of its own. Not asserting the exact parent — the model may call the tool more + // than once, and the span that ran the loader is not reliably the one found here. expect(getSpanOp(dataloaderSpan!)).toBe('cache.get'); - expect(dataloaderSpan?.trace_id).toBe(executeTool?.trace_id); - expect(dataloaderSpan?.parent_span_id).toBe(executeTool?.span_id); + expect(dataloaderSpan?.trace_id).toBe(toolSpan?.trace_id); }); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts index 11652d339618..f8b59be4fc1b 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/errors.test.ts @@ -1,6 +1,6 @@ import { expect, test } from '@playwright/test'; -import { collectStreamedSpans, getSpanOp, waitForError } from '@sentry-internal/test-utils'; -import { runAgentTurn } from './utils'; +import { collectStreamedSpans, waitForError } from '@sentry-internal/test-utils'; +import { newConversationId, runAgentTurn } from './utils'; const APP = 'node-flue'; @@ -10,10 +10,10 @@ test('captures an error thrown inside a Flue tool and marks its span errored', a event => event.exception?.values?.[0]?.value === 'Intentional flue tool failure', ); const spansPromise = collectStreamedSpans(APP, spansOfTrace => - spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool'), + spansOfTrace.some(span => span.attributes?.['gen_ai.tool.name']?.value === 'fail_now'), ); - await runAgentTurn(baseURL!, 'failure-conversation', 'Please call fail_now to trigger a failure.'); + await runAgentTurn(baseURL!, newConversationId('failure'), 'Please call fail_now to trigger a failure.'); const error = await errorPromise; expect(error.exception?.values?.[0]?.value).toBe('Intentional flue tool failure'); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts index be32a4c90b52..ccf3e552a87d 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/flue.test.ts @@ -1,17 +1,28 @@ import { expect, test } from '@playwright/test'; import { collectStreamedSpans, getSpanOp } from '@sentry-internal/test-utils'; -import { runAgentTurn } from './utils'; +import { newConversationId, runAgentTurn } from './utils'; const APP = 'node-flue'; -const hasOps = (ops: string[]) => (spansOfTrace: { attributes?: Record }[]) => +type SpanLike = { name?: string; attributes?: Record }; + +const hasOps = (ops: string[]) => (spansOfTrace: SpanLike[]) => ops.every(op => spansOfTrace.some(span => getSpanOp(span) === op)); +// Anchored on the tool the test drives, so a leftover trace from another test cannot satisfy it. +const usedTool = (toolName: string) => (spansOfTrace: SpanLike[]) => + spansOfTrace.some(span => span.attributes?.['gen_ai.tool.name']?.value === toolName); + test('captures the invoke_agent / chat / execute_tool hierarchy for a Flue turn', async ({ baseURL }) => { // The trace flushes across several envelopes, so accumulate it rather than asserting on one. - const spansPromise = collectStreamedSpans(APP, hasOps(['gen_ai.invoke_agent', 'gen_ai.chat', 'gen_ai.execute_tool'])); + const spansPromise = collectStreamedSpans( + APP, + spansOfTrace => + hasOps(['gen_ai.invoke_agent', 'gen_ai.chat', 'gen_ai.execute_tool'])(spansOfTrace) && + usedTool('get_weather')(spansOfTrace), + ); - await runAgentTurn(baseURL!, 'weather-conversation', 'What is the weather in Paris?'); + await runAgentTurn(baseURL!, newConversationId('weather'), 'What is the weather in Paris?'); const spans = await spansPromise; const invokeAgent = spans.find(span => getSpanOp(span) === 'gen_ai.invoke_agent'); @@ -40,12 +51,10 @@ test('captures the invoke_agent / chat / execute_tool hierarchy for a Flue turn' test('nests a manual span raised inside a tool under that tool span', async ({ baseURL }) => { const spansPromise = collectStreamedSpans( APP, - spansOfTrace => - spansOfTrace.some(span => getSpanOp(span) === 'gen_ai.execute_tool') && - spansOfTrace.some(span => span.name === 'resolve-weather'), + spansOfTrace => usedTool('get_weather')(spansOfTrace) && spansOfTrace.some(span => span.name === 'resolve-weather'), ); - await runAgentTurn(baseURL!, 'manual-span-conversation', 'What is the weather in Berlin?'); + await runAgentTurn(baseURL!, newConversationId('manual-span'), 'What is the weather in Berlin?'); const spans = await spansPromise; const executeTool = spans.find(span => getSpanOp(span) === 'gen_ai.execute_tool'); @@ -66,7 +75,7 @@ test('nests the provider HTTP call inside the chat span', async ({ baseURL }) => spansOfTrace.some(span => getSpanOp(span) === 'http.client'), ); - await runAgentTurn(baseURL!, 'provider-http-conversation', 'Say hello.'); + await runAgentTurn(baseURL!, newConversationId('provider-http'), 'Say hello.'); const spans = await spansPromise; const chat = spans.find(span => getSpanOp(span) === 'gen_ai.chat'); diff --git a/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts b/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts index e214f4abcea6..02af2469f47b 100644 --- a/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts +++ b/dev-packages/e2e-tests/test-applications/node-flue/tests/utils.ts @@ -1,5 +1,16 @@ import { expect } from '@playwright/test'; +/** + * A conversation id nothing has used yet. + * + * `runAgentTurn` waits for the conversation to report a settlement, so a fixed id that already has + * one — a Playwright retry, or the `test:dev` run hitting the record `test:prod` left behind — + * would return before the new turn finished. + */ +export function newConversationId(prefix: string): string { + return `${prefix}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; +} + /** * Run one agent turn over Flue's agent router and wait for it to settle. *