🤖 feat: add opt-in AgentPond tracing

.gitignore

@@ -85,6 +85,9 @@ docs/mermaid.min.js
 # AI SDK DevTools (local dev only)
 .devtools
 
+# AgentPond local trace environments
+.agentpond/
+
 # Nix
 .envrc
 .direnv

docs/reference/telemetry.mdx

@@ -46,6 +46,17 @@ MUX_DISABLE_TELEMETRY=1 mux
 
 This disables telemetry collection at the backend level.
 
+## Optional AI tracing
+
+Mux can export OpenInference traces for agent turns to an
+[AgentPond](https://github.com/marcusschiesser/agentpond) Files SDK environment.
+This is separate from anonymous usage telemetry and remains disabled unless
+`FILES_SDK_PROVIDER` and the selected provider's environment variables are set
+before Mux starts.
+
+Prompt and response content is never included in these traces. They contain
+operation, model, timing, and token-usage metadata.
+
 ## Source code
 
 - **Payload definitions**: [`src/common/telemetry/payload.ts`](https://github.com/coder/mux/blob/main/src/common/telemetry/payload.ts)

package.json

@@ -49,6 +49,8 @@
   "dependencies": {
     "@1password/sdk": "^0.4.0",
     "@agentclientprotocol/sdk": "^0.25.0",
+    "@agentpond/files-sdk": "0.2.0",
+    "@agentpond/otel": "0.1.5",
     "@ai-sdk/amazon-bedrock": "^5.0.15",
     "@ai-sdk/anthropic": "^4.0.11",
     "@ai-sdk/deepseek": "^3.0.7",
@@ -57,7 +59,9 @@
     "@ai-sdk/moonshotai": "^3.0.15",
     "@ai-sdk/openai": "^4.0.11",
     "@ai-sdk/openai-compatible": "^3.0.7",
+    "@ai-sdk/otel": "1.0.19",
     "@ai-sdk/xai": "^4.0.10",
+    "@arizeai/openinference-vercel": "3.1.0",
     "@aws-sdk/credential-providers": "^3.940.0",
     "@dnd-kit/core": "^6.3.1",
     "@dnd-kit/sortable": "^10.0.0",
@@ -69,6 +73,8 @@
     "@mozilla/readability": "^0.6.0",
     "@novnc/novnc": "^1.6.0",
     "@openrouter/ai-sdk-provider": "^3.0.0",
+    "@opentelemetry/api": "1.9.1",
+    "@opentelemetry/sdk-trace-node": "2.10.0",
     "@orpc/client": "^1.11.3",
     "@orpc/openapi": "^1.12.2",
     "@orpc/server": "^1.11.3",
@@ -104,6 +110,7 @@
     "electron-updater": "^6.6.2",
     "electron-window-state": "^5.0.3",
     "express": "^5.1.0",
+    "files-sdk": "2.2.2",
     "fix-path": "5.0.0",
     "ghostty-web": "^0.4.0-next.7.g03ead6e",
     "ignore": "^7.0.5",

src/node/services/agentPondTracing.ts

@@ -0,0 +1,75 @@
+import { createFilesSpanExporterFromRuntimeEnv } from "@agentpond/files-sdk/otel";
+import { OpenTelemetry } from "@ai-sdk/otel";
+import {
+  isOpenInferenceSpan,
+  OpenInferenceBatchSpanProcessor,
+} from "@arizeai/openinference-vercel";
+import { NodeTracerProvider } from "@opentelemetry/sdk-trace-node";
+import type { TelemetryOptions } from "ai";
+import { log } from "./log";
+
+const AGENTPOND_TRACING_STATE = Symbol.for("mux.agentpond-tracing");
+
+interface AgentPondTracingState {
+  provider: NodeTracerProvider;
+  telemetry: TelemetryOptions;
+}
+
+type AgentPondGlobal = typeof globalThis & {
+  [AGENTPOND_TRACING_STATE]?: AgentPondTracingState;
+};
+
+function createAgentPondTracingState(): AgentPondTracingState | undefined {
+  // Agent traces can contain operational metadata, so users must explicitly
+  // provide a Files SDK environment before Mux creates an exporter.
+  if (!process.env.FILES_SDK_PROVIDER) {
+    return undefined;
+  }
+
+  try {
+    const provider = new NodeTracerProvider({
+      spanProcessors: [
+        new OpenInferenceBatchSpanProcessor({
+          exporter: createFilesSpanExporterFromRuntimeEnv(),
+          spanFilter: isOpenInferenceSpan,
+          reparentOrphanedSpans: true,
+        }),
+      ],
+    });
+
+    return {
+      provider,
+      telemetry: {
+        functionId: "mux-agent",
+        integrations: [
+          new OpenTelemetry({
+            tracer: provider.getTracer("mux"),
+            usage: true,
+          }),
+        ],
+        isEnabled: true,
+        recordInputs: false,
+        recordOutputs: false,
+      },
+    };
+  } catch (error) {
+    // Optional tracing must never stop Mux from starting or serving an AI turn.
+    log.warn("Failed to initialize AgentPond tracing", { error });
+    return undefined;
+  }
+}
+
+const agentPondGlobal = globalThis as AgentPondGlobal;
+const tracingState =
+  agentPondGlobal[AGENTPOND_TRACING_STATE] ??
+  (agentPondGlobal[AGENTPOND_TRACING_STATE] = createAgentPondTracingState());
+
+export const agentPondTelemetry = tracingState?.telemetry;
+
+export async function flushAgentPondTracing(): Promise<void> {
+  try {
+    await tracingState?.provider.forceFlush();
+  } catch (error) {
+    log.warn("Failed to flush AgentPond tracing", { error });
+  }
+}

src/node/services/agentSkills/builtInSkillContent.generated.ts

@@ -6690,6 +6690,17 @@ export const BUILTIN_SKILL_FILES: Record<string, Record<string, string>> = {
       "",
       "This disables telemetry collection at the backend level.",
       "",
+      "## Optional AI tracing",
+      "",
+      "Mux can export OpenInference traces for agent turns to an",
+      "[AgentPond](https://github.com/marcusschiesser/agentpond) Files SDK environment.",
+      "This is separate from anonymous usage telemetry and remains disabled unless",
+      "`FILES_SDK_PROVIDER` and the selected provider's environment variables are set",
+      "before Mux starts.",
+      "",
+      "Prompt and response content is never included in these traces. They contain",
+      "operation, model, timing, and token-usage metadata.",
+      "",
       "## Source code",
       "",
       "- **Payload definitions**: [`src/common/telemetry/payload.ts`](https://github.com/coder/mux/blob/main/src/common/telemetry/payload.ts)",

src/node/services/streamManager.ts

@@ -89,6 +89,7 @@ import { classify429Capacity } from "@/common/utils/errors/classify429Capacity";
 import { extractChunkDeltaText } from "@/common/utils/ai/streamChunks";
 import { PROVIDER_DEFINITIONS } from "@/common/constants/providers";
 import { isRefusalFinishReason } from "@/common/utils/messages/refusalFinishReason";
+import { agentPondTelemetry } from "./agentPondTracing";
 
 // Disable noisy AI SDK warning logging.
 globalThis.AI_SDK_LOG_WARNINGS = false;
@@ -1858,6 +1859,7 @@ export class StreamManager extends EventEmitter {
       providerOptions: request.providerOptions as any, // Pass provider-specific options (thinking/reasoning config)
       headers: request.headers, // Per-request HTTP headers (e.g., anthropic-beta for 1M context)
       maxOutputTokens: request.maxOutputTokens,
+      ...(agentPondTelemetry !== undefined ? { telemetry: agentPondTelemetry } : {}),
       ...(request.streamCallSettings ?? {}),
     });
   }