[translation-bench] Part 1/n: scaffold + generate price snapshots + catalogs

ts/packages/benchmarks/.gitignore

@@ -0,0 +1,5 @@
+node_modules/
+dist/
+data/
+results/
+*.tsbuildinfo

ts/packages/benchmarks/LICENSE

@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE

ts/packages/benchmarks/README.AUTOGEN.md

@@ -0,0 +1,43 @@
+<!-- Copyright (c) Microsoft Corporation. -->
+<!-- Licensed under the MIT License. -->
+
+<!-- AUTOGEN:DOCS:START -->
+
+<!-- AUTOGEN:DOCS:HASH:sha256=8d1900f57cb0c7fbbee4ead35199a1e9facb6197211014e9ee74c87b4480f21a -->
+<!-- AUTOGEN:DOCS:SOURCE: ./README.md (hand-written documentation; this file is the AI-generated companion) -->
+
+# @typeagent/benchmarks — AI-generated documentation
+
+> 📝 **Placeholder documentation — not yet AI-authored.** Re-run `pnpm docs:generate:llm --package benchmarks` to populate this file, or read [`./README.md`](./README.md) for the hand-written documentation in the meantime. The deterministic Reference section below is already populated.
+
+## Overview
+
+TypeAgent translation benchmark (scaffold)
+
+## Reference
+
+> ⚙️ **Auto-generated, no AI involvement.** Built deterministically from `package.json`, `src/`, and the workspace dependency graph at the commit recorded in the staleness footer at the end of this file. Hand edits to this file will be overwritten on the next run.
+
+### Entry points
+
+_No public exports declared in `package.json`._
+
+### Dependencies
+
+Workspace:
+
+- [@typeagent/agent-sdk](../../packages/agentSdk/README.md)
+- [agent-dispatcher](../../packages/dispatcher/dispatcher/README.md)
+- [default-agent-provider](../../packages/defaultAgentProvider/README.md)
+
+External: _None at runtime._
+
+### Files of interest
+
+`./src/core/model-prices.generated.json`, `./src/core/paths.ts`, `./src/core/prices.ts`, …and 5 more under `./src/`.
+
+---
+
+_Auto-generated against commit `3bc153f3a186261d47ef517dc8cd0af4c161092b` on `2026-07-31T17:39:11.834Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
+
+<!-- AUTOGEN:DOCS:END -->

ts/packages/benchmarks/README.md

@@ -0,0 +1,11 @@
+# @typeagent/benchmarks
+
+Action-translation eval for TypeAgent: pinned catalogs, model prices, and scoring harness.
+
+## Trademarks
+
+This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft
+trademarks or logos is subject to and must follow
+[Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
+Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
+Any use of third-party trademarks or logos are subject to those third-party's policies.

ts/packages/benchmarks/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@typeagent/benchmarks",
+  "version": "0.0.1",
+  "description": "TypeAgent translation benchmark (scaffold)",
+  "homepage": "https://github.com/microsoft/TypeAgent#readme",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/microsoft/TypeAgent.git",
+    "directory": "ts/packages/benchmarks"
+  },
+  "license": "MIT",
+  "author": "Microsoft",
+  "type": "module",
+  "scripts": {
+    "build": "tsc -b && node ./scripts/copyAssets.mjs",
+    "clean": "node ./scripts/clean.mjs",
+    "gen-catalog": "pnpm run build && node dist/translationBench/scripts/genCatalog.js",
+    "prettier": "prettier --check package.json tsconfig.json src scripts --ignore-path ../../.prettierignore",
+    "prettier:fix": "prettier --write package.json tsconfig.json src scripts --ignore-path ../../.prettierignore",
+    "snapshot-prices": "pnpm run build && node dist/translationBench/scripts/snapshotPrices.js",
+    "test": "pnpm run build",
+    "test:local": "pnpm run test",
+    "tsc": "tsc -b"
+  },
+  "dependencies": {
+    "@typeagent/agent-sdk": "workspace:*",
+    "agent-dispatcher": "workspace:*",
+    "default-agent-provider": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "prettier": "^3.5.3",
+    "typescript": "~5.4.5"
+  }
+}

ts/packages/benchmarks/scripts/clean.mjs

@@ -0,0 +1,11 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { rmSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+// Drop emit + incremental state so the next `tsc -b` fully rebuilds.
+rmSync(path.join(root, "dist"), { recursive: true, force: true });
+rmSync(path.join(root, "tsconfig.tsbuildinfo"), { force: true });

ts/packages/benchmarks/scripts/copyAssets.mjs

@@ -0,0 +1,27 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/** Copy non-TS assets next to compiled output (tsc does not emit .json). */
+import { copyFileSync, mkdirSync } from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+
+const assets = [
+    [
+        "src/translationBench/catalog.generated.json",
+        "dist/translationBench/catalog.generated.json",
+    ],
+    [
+        "src/core/model-prices.generated.json",
+        "dist/core/model-prices.generated.json",
+    ],
+];
+
+for (const [fromRel, toRel] of assets) {
+    const from = path.join(root, fromRel);
+    const to = path.join(root, toRel);
+    mkdirSync(path.dirname(to), { recursive: true });
+    copyFileSync(from, to);
+}

ts/packages/benchmarks/src/core/model-prices.generated.json

@@ -0,0 +1,58 @@
+{
+  "source": "https://models.dev/api.json",
+  "version": "2026-07-31T02:08:45.776Z",
+  "unit": "USD per 1M tokens",
+  "provider": "openai",
+  "rates": {
+    "gpt-4.1": {
+      "inUsdPer1M": 2,
+      "cachedUsdPer1M": 0.5,
+      "outUsdPer1M": 8
+    },
+    "gpt-4.1-mini": {
+      "inUsdPer1M": 0.4,
+      "cachedUsdPer1M": 0.1,
+      "outUsdPer1M": 1.6
+    },
+    "gpt-5": {
+      "inUsdPer1M": 1.25,
+      "cachedUsdPer1M": 0.125,
+      "outUsdPer1M": 10
+    },
+    "gpt-5-mini": {
+      "inUsdPer1M": 0.25,
+      "cachedUsdPer1M": 0.025,
+      "outUsdPer1M": 2
+    },
+    "gpt-5.4": {
+      "inUsdPer1M": 2.5,
+      "cachedUsdPer1M": 0.25,
+      "outUsdPer1M": 15
+    },
+    "gpt-5.4-mini": {
+      "inUsdPer1M": 0.75,
+      "cachedUsdPer1M": 0.075,
+      "outUsdPer1M": 4.5
+    },
+    "gpt-5.4-nano": {
+      "inUsdPer1M": 0.2,
+      "cachedUsdPer1M": 0.02,
+      "outUsdPer1M": 1.25
+    },
+    "gpt-5.6-luna": {
+      "inUsdPer1M": 1,
+      "cachedUsdPer1M": 0.1,
+      "outUsdPer1M": 6
+    },
+    "gpt-5.6-sol": {
+      "inUsdPer1M": 5,
+      "cachedUsdPer1M": 0.5,
+      "outUsdPer1M": 30
+    },
+    "gpt-5.6-terra": {
+      "inUsdPer1M": 2.5,
+      "cachedUsdPer1M": 0.25,
+      "outUsdPer1M": 15
+    }
+  }
+}

ts/packages/benchmarks/src/core/paths.ts

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+// Same pattern as cache/defaultAgentProvider: this file emits to dist/core/,
+// so ../.. is the package root.
+const packageRoot = path.join("..", "..");
+
+/** Resolve a path relative to the package root (works from compiled dist/). */
+export function getPackageFilePath(packageRootRelativePath: string): string {
+    if (path.isAbsolute(packageRootRelativePath)) {
+        return packageRootRelativePath;
+    }
+    return fileURLToPath(
+        new URL(
+            path.join(packageRoot, packageRootRelativePath),
+            import.meta.url,
+        ),
+    );
+}
+
+export const packageRootAbs = getPackageFilePath(".");
+
+/** datasets: <package>/data/translationBench/ */
+export function dataDir(): string {
+    return getPackageFilePath(path.join("data", "translationBench"));
+}
+
+/** run output: <package>/results/translationBench/ */
+export function resultsDir(): string {
+    return getPackageFilePath(path.join("results", "translationBench"));
+}

ts/packages/benchmarks/src/core/prices.ts

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { createRequire } from "node:module";
+import type { Prices } from "./types.js";
+
+const require = createRequire(import.meta.url);
+
+export interface PriceTable {
+    source: string;
+    version: string;
+    unit?: string;
+    rates: Record<string, Prices>;
+}
+
+export const PRICES: PriceTable =
+    require("./model-prices.generated.json") as PriceTable;
+
+export function pricesFor(model: string): {
+    prices: Prices | undefined;
+    table: PriceTable;
+} {
+    const table: PriceTable = { ...PRICES, rates: { ...PRICES.rates } };
+    const prices = table.rates[model] ?? table.rates[model.replace(/\*$/, "")];
+    return { prices, table };
+}

ts/packages/benchmarks/src/core/types.ts

@@ -0,0 +1,49 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+/** Single translated / expected action. */
+export interface Action {
+    schemaName: string;
+    actionName: string;
+    parameters: unknown;
+}
+
+/** How multi-action expectations are ordered. */
+export type Order = "strict" | "any";
+
+/** One eval row (dataset case). */
+export interface EvalCase {
+    id: string;
+    utterance: string;
+    expectedActions: Action[];
+    order: Order;
+    /** Optional prior turns; shape filled in when the runner lands. */
+    history?: unknown;
+}
+
+/** Token cost rates (USD per 1M tokens). */
+export interface Prices {
+    inUsdPer1M: number;
+    cachedUsdPer1M: number;
+    outUsdPer1M: number;
+}
+
+/** Accumulated usage for one request or run. */
+export interface CostRecord {
+    prompt: number;
+    completion: number;
+    cached: number | undefined;
+    reasoning: number | undefined;
+    model: string;
+    usageCalls: number;
+    estimatedCostUsd: number | undefined;
+}
+
+/** One action in the pinned catalog. */
+export interface CatalogEntry {
+    schemaName: string;
+    actionName: string;
+    parameters: string;
+    paramSpec?: unknown;
+    description?: string;
+}