diff --git a/.cargo/config.toml b/.cargo/config.toml index 7b7b2eb4e1f..8c46f31e713 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,6 +3,7 @@ rustflags = ["--cfg", "tokio_unstable"] [alias] llm = "run --package xtask-llm-benchmark --bin llm_benchmark --" +stack-bench = "run --package xtask-stack-bench --bin stack_bench --" ci = "run -p ci --" regen = "run -p regen --" smoketest = "ci smoketests --" diff --git a/Cargo.lock b/Cargo.lock index 2fca17a44f6..8b5a39cfdd2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11578,6 +11578,15 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "xtask-stack-bench" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.50", + "serde_json", +] + [[package]] name = "xxhash-rust" version = "0.8.15" diff --git a/Cargo.toml b/Cargo.toml index 939ba33a46c..e4230319eaf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ members = [ "tools/release", "tools/regen", "tools/xtask-llm-benchmark", + "tools/stack-bench/xtask", "crates/bindings-typescript/test-app/server", "crates/bindings-typescript/test-react-router-app/server", "crates/bindings-typescript/test-solid-router/server", diff --git a/tools/stack-bench/.gitignore b/tools/stack-bench/.gitignore new file mode 100644 index 00000000000..cd30d481f39 --- /dev/null +++ b/tools/stack-bench/.gitignore @@ -0,0 +1,13 @@ +# Harness copies injected into each task by `cargo stack-bench build` (regenerated, not source). +tasks/*/*/tests/harness/ + +# Build artifacts. +**/node_modules/ +**/dist/ +**/target/ + +# Convex generated bindings. +**/convex/_generated/ + +# Harbor job results (cargo stack-bench all / -o). +jobs/ diff --git a/tools/stack-bench/README.md b/tools/stack-bench/README.md new file mode 100644 index 00000000000..4a8b2b967ce --- /dev/null +++ b/tools/stack-bench/README.md @@ -0,0 +1,187 @@ +# stack-bench + +A cross-backend, **agentic** benchmark: rank backends (SpacetimeDB vs Convex vs +Supabase vs …) by how successfully and efficiently an AI coding agent builds +real-time apps on each. Built in [Harbor](https://www.harborframework.com/) +format (the harness behind Terminal-Bench 2.0), so every task is a standard, +publishable Harbor task with fixed prompts and machine verification. + +This is **separate from** the one-shot leaderboard in `tools/xtask-llm-benchmark/` +(which powers spacetimedb.com/llms-benchmark and ranks *models* on SpacetimeDB +only). stack-bench fixes the agent+model and varies the **backend**. + +## Tasks + +| Task | What it tests | Status | +|---|---|---| +| `team-chat` | **The main benchmark.** Full team-chat backend: rooms/owners/membership, presence, server-maintained unread counters, message edit + tombstone delete, idempotent sends, per-room **gapless seq under concurrency**, atomic credit tips, **restart durability**, real-time push (fan-out, late-joiner, edits/deletes/presence/membership/balances), latency + throughput. 46 checks, 4 weighted metric groups. | **oracle 1.0 on spacetimedb + convex, through Harbor** | +| `realtime-chat` | The original spike: minimal single-room chat, 4 checks. Kept as a smoke test. | oracle 1.0 on both, via Harbor | + +Each `tasks///` is a complete Harbor task (instruction.md + +task.toml + environment/ + solution/ + tests/). + +## Design (what makes team-chat hard to saturate) + +The verifier (`_shared/harness/`) is one backend-neutral TypeScript scenario +written against an `AppClient` contract (`src/appClient.ts`); each backend ships +a thin adapter implementing that contract with its **real client SDK**. Grading +is purely behavioral — multiple concurrent SDK clients drive the deployed app. + +Four weighted metric groups (also emitted individually in `reward.json`): + +- **correctness 0.40** — functional rules + transactional behavior under + concurrency: gapless per-room seq with 3 concurrent writers, exact unread + counters (multi-row atomicity with the message insert), tip conservation + under concurrent transfers, idempotent resends, monotone mark-read, + tombstone privacy, permission edges (kick/leave/edit/delete), and a final + τ-bench-style **full goal-state comparison** against the harness's model. +- **realtime 0.30** — push, not polling: 3-subscriber fan-out exactly-once in + seq order, cross-room isolation (server-side subscription filtering), + late-joiner history-then-live with no dupes/gaps at the boundary, live + propagation of edits, deletes, presence, membership, and balances. +- **durability 0.20** — Jepsen-style process kill + restart mid-scenario via + each environment's `/opt/stack-bench/backendctl restart`; verifies every + piece of state survives (messages incl. edits/tombstones, memberships, + read state, balances), that the per-room **seq counter continues gapless** + (durable counter — in-memory counters that reset fail), that the + `client_msg_id` dedupe record survives, and that real-time works again. +- **perf 0.10** — delivery latency p95 under a generous threshold (1.5s) and a + 120-message concurrent burst delivered within budget; raw p50/p95 and + throughput are reported as metrics either way. + +Why an agent can't trivially score 1.0: the concurrency checks require real +transactional design (read-increment-write counters, multi-row atomic +updates), the durability checks kill lazy in-memory state, the late-joiner +boundary and exactly-once fan-out catch sloppy subscription logic, and the +scenario's ~46 checks are graded independently — partial credit makes the +leaderboard discriminating rather than binary. + +The harness writes (Harbor contract, all under `/logs/verifier/`): + +- `reward.txt` — the scalar weighted reward +- `reward.json` — named metrics: group subscores + latency/throughput numbers +- `result.json` — every check with pass/fail + failure detail: the + machine-readable findings payload for a multi-step agent feedback loop + +## Layout + +``` +stack-bench/ +├── _shared/ +│ ├── team-chat.base.md backend-agnostic team-chat spec (source of truth) +│ ├── instruction.base.md realtime-chat spec (spike) +│ └── harness/ the shared grader (TS via tsx; injected into tests/) +│ └── src/ +│ ├── appClient.ts team-chat cross-backend contract +│ ├── teamChat/ model.ts (goal state) + scenario.ts (~46 checks) +│ ├── runTeamChat.ts entry point: reward.txt/reward.json/result.json +│ └── … chatClient.ts/scenario.ts/runScenario.ts (spike) +├── tasks/ +│ ├── team-chat/ +│ │ ├── spacetimedb/ env: rust+node+spacetime, backendctl; oracle Rust module +│ │ └── convex/ env: convex-backend image + node, backendctl; oracle app +│ └── realtime-chat/ the original spike tasks +├── xtask/ cargo runner — `cargo stack-bench …` +└── README.md +``` + +Instruction files are assembled as: shared base spec + backend-specific +**Contract** section pinning the exact identifiers the grader connects to +(SpacetimeDB: table schemas + reducer signatures; Convex: function names + +arg/return shapes). Fixed prompts, machine-checkable surface. + +## Environments & the restart hook + +Every backend runs **in the agent's container** (single-container model): + +- `spacetimedb`: `spacetime start` backgrounded at boot. +- `convex`: the official `ghcr.io/get-convex/convex-backend` image as the base + (Ubuntu 24.04 + its `run_backend.sh`), with Node 22 installed on top; + deterministic admin key from a fixed instance name/secret. + +Both ship `/opt/stack-bench/backendctl` (`start|stop|restart|wait-ready`). +Boot and the verifier's durability restart use the same script, so restart +behaves exactly like a fresh boot against the same data dir. Single-container +also means the backend shares the agent's cpu/memory budget (`task.toml +[environment]`), keeping the perf metrics a fair cross-backend comparison. + +## How to run + +```bash +# Inject the shared grader into each task's tests/ (auto-run by the commands below). +cargo stack-bench build + +# List task variants. +cargo stack-bench list + +# Oracle sanity check (expect reward 1.0). --task defaults to team-chat. +cargo stack-bench oracle spacetimedb +cargo stack-bench oracle convex +cargo stack-bench oracle spacetimedb --task realtime-chat + +# Run a real agent (needs the provider API key for the agent). +cargo stack-bench agent spacetimedb --model anthropic/claude-opus-4-6 +cargo stack-bench agent convex --model anthropic/claude-opus-4-6 + +# Run EVERY task/backend with the same agent+model and print a comparison table. +cargo stack-bench all --agent claude-code --model anthropic/claude-opus-4-6 + +# Anything after `--` is forwarded verbatim to `harbor run` (e.g. -k 5 for pass^k trials): +cargo stack-bench agent spacetimedb --model … -- -k 5 +``` + +The tasks are plain Harbor tasks, so they also run without cargo: +`harbor run -p tasks/team-chat/spacetimedb -a oracle -y`. + +Local tooling used to validate: `harbor 0.7.1`, `spacetime 2.5.0`, `node v22`, +`docker`, `cargo`. + +## Agents & models + +Harbor's `-a` flag picks who attempts the task: `oracle` runs the committed +reference solution (harness self-test, no API key), `nop` is the empty +baseline, and real agents (`claude-code`, `codex`, `terminus`, …) attempt the +task from `instruction.md` with `-m provider/model`. See Harbor's docs for +keys; OpenRouter works for the model-routed agents. + +## Multi-step feedback loop + +`result.json` lists every failed check with a concrete detail string +(machine-readable findings). A driver can hand these back to the agent and +re-run the verifier for a fix-it loop; per-run effort (tokens, cost, +wall-clock) comes from Harbor's job output. Wiring a standard loop driver is +future work (Harbor `[[steps]]` semantics are still being validated). + +## Validation status + +- **team-chat via Harbor** (Dockerized env + SHARED verifier + backendctl + restart): oracle reward **1.0000 on both backends** — 46/46 checks, 0 + exceptions. `harbor run -p tasks/team-chat/{spacetimedb,convex} -a oracle`. +- Observed oracle metrics (same host, same container limits): + SpacetimeDB p50 4ms / p95 7ms / 952 msg/s burst; + Convex p50 18ms / p95 27ms / ~90 msg/s burst. +- **realtime-chat via Harbor**: oracle 1.0 on both (spike result, June 2026; + pinned to the older CLI/SDK it was built against). + +## Anti-gaming notes (from the design research) + +- Grading is behavioral through real SDKs against the running app; the agent + never sees `tests/` (Harbor copies it in at verification time). +- Durability checks make hardcoded/stubbed backends fail: state must survive + a process kill, and counters must continue exactly. +- For real leaderboard runs, pre-bake dependencies and cut agent internet + access after setup (SWE-bench-style retrieval gaming), and pin image + digests + tool versions for reproducibility. +- Publish tasks, graders, and oracle solutions; report results honestly + including where SpacetimeDB loses. + +## Remaining work + +- Harbor-mode validation of team-chat (in progress) and real-agent runs. +- Effort metrics (tokens/cost/turns) surfaced in the comparison table + (`in_tok`/`out_tok` come from Harbor job output today). +- Feedback-loop driver (Harbor `[[steps]]` or a thin outer loop). +- More backends (Postgres, Mongo, Supabase, Firebase) — each needs an + environment + backendctl, an adapter (~200 lines), an instruction Contract + section, and an oracle solution. +- Pre-baked deps in the environment images for speed + hermeticity. diff --git a/tools/stack-bench/_shared/harness/package-lock.json b/tools/stack-bench/_shared/harness/package-lock.json new file mode 100644 index 00000000000..87fe673d5cc --- /dev/null +++ b/tools/stack-bench/_shared/harness/package-lock.json @@ -0,0 +1,47 @@ +{ + "name": "@stack-bench/harness", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@stack-bench/harness", + "version": "0.0.1", + "devDependencies": { + "@types/node": "^20.0.0", + "typescript": "^5.4.0" + } + }, + "node_modules/@types/node": { + "version": "20.19.41", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.19.41.tgz", + "integrity": "sha512-ECymXOukMnOoVkC2bb1Vc/w/836DXncOg5m8Xj1RH7xSHZJWNYY6Zh7EH477vcnD5egKNNfy2RpNOmuChhFPgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, + "node_modules/typescript": { + "version": "5.9.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", + "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/tools/stack-bench/_shared/harness/package.json b/tools/stack-bench/_shared/harness/package.json new file mode 100644 index 00000000000..26113d5d1c7 --- /dev/null +++ b/tools/stack-bench/_shared/harness/package.json @@ -0,0 +1,15 @@ +{ + "name": "@stack-bench/harness", + "version": "0.0.1", + "private": true, + "type": "module", + "description": "Backend-neutral behavioral grader for stack-bench tasks. Compiled to dist/ and injected into each task's tests/ by build-task.sh.", + "main": "dist/runScenario.js", + "scripts": { + "build": "tsc -p tsconfig.json" + }, + "devDependencies": { + "typescript": "^5.4.0", + "@types/node": "^20.0.0" + } +} diff --git a/tools/stack-bench/_shared/harness/src/appClient.ts b/tools/stack-bench/_shared/harness/src/appClient.ts new file mode 100644 index 00000000000..11a2ab098d6 --- /dev/null +++ b/tools/stack-bench/_shared/harness/src/appClient.ts @@ -0,0 +1,85 @@ +// The cross-backend contract for the `team-chat` task. +// +// Every backend ships an *adapter* implementing this interface with that +// backend's REAL real-time client SDK. The behavioral scenario is written only +// against this interface, so the identical grader runs on every backend. +// +// Semantics the adapter must honor (the scenario depends on these): +// - Mutation methods resolve when the backend ACCEPTS the operation and +// REJECT (throw) when the backend refuses it (validation/permission error). +// - subscribeRoom delivers the room's full message history as `message` events +// (in seq order, reflecting current edited/deleted state), then live events +// thereafter. Message edits/deletes arrive as `message` events for the same +// clientMsgId with updated fields. +// - subscribeUsers mirrors the user table: one `user` event per user at +// subscribe time, then one per change (status/balance) thereafter. +// - subscribeMembers mirrors a room's membership: `member` events on join / +// read-state change, and a `memberRemoved` event on leave/kick. + +export interface UserRecord { + username: string; + status: string; // "online" | "away" | "offline" + balance: number; +} + +export interface MessageRecord { + seq: number; + clientMsgId: string; + sender: string; + text: string; + edited: boolean; + deleted: boolean; +} + +export interface MemberRecord { + user: string; + lastReadSeq: number; + unread: number; +} + +export interface RoomEventHandlers { + onMessage: (msg: MessageRecord) => void; +} + +export interface UserEventHandlers { + onUser: (user: UserRecord) => void; +} + +export interface MemberEventHandlers { + onMember: (member: MemberRecord) => void; + onMemberRemoved?: (user: string) => void; +} + +export interface AppClient { + /** Establish a real-time connection. Rejects if the backend is unreachable. */ + connect(): Promise; + /** Tear down the connection (idempotent). */ + close(): Promise; + + // ---- mutations (resolve on accept, throw on reject) ---- + register(username: string): Promise; + setStatus(username: string, status: string): Promise; + createRoom(username: string, room: string): Promise; + joinRoom(username: string, room: string): Promise; + leaveRoom(username: string, room: string): Promise; + kick(actor: string, room: string, target: string): Promise; + sendMessage(sender: string, room: string, text: string, clientMsgId: string): Promise; + editMessage(actor: string, room: string, clientMsgId: string, newText: string): Promise; + deleteMessage(actor: string, room: string, clientMsgId: string): Promise; + markRead(user: string, room: string, upToSeq: number): Promise; + tip(fromUser: string, toUser: string, amount: number): Promise; + + // ---- subscriptions (push; history-then-live) ---- + subscribeRoom(room: string, handlers: RoomEventHandlers): Promise; + subscribeUsers(handlers: UserEventHandlers): Promise; + subscribeMembers(room: string, handlers: MemberEventHandlers): Promise; + + // ---- point-in-time snapshot queries (used for end-state verification) ---- + getUser(username: string): Promise; + getRoomOwner(room: string): Promise; + getMembers(room: string): Promise; + getMessages(room: string): Promise; // seq ascending +} + +/** Each adapter module must default-export a factory that builds a fresh client. */ +export type AppClientFactory = () => AppClient; diff --git a/tools/stack-bench/_shared/harness/src/chatClient.ts b/tools/stack-bench/_shared/harness/src/chatClient.ts new file mode 100644 index 00000000000..d275c875697 --- /dev/null +++ b/tools/stack-bench/_shared/harness/src/chatClient.ts @@ -0,0 +1,34 @@ +// The cross-backend contract. +// +// Every backend (SpacetimeDB, Convex, Supabase, ...) ships an *adapter* that +// implements this interface using that backend's REAL real-time client SDK. +// The behavioral scenario (scenario.ts) is written ONLY against this interface, +// so the exact same test grades every backend identically. This is what makes +// stack-bench a fair cross-backend comparison rather than N bespoke graders. + +export interface ChatMessage { + sender: string; + text: string; +} + +export interface ChatClient { + /** Establish a real-time connection to the running app. */ + connect(): Promise; + + /** + * Subscribe to the room's message stream. `onMessage` MUST fire: + * - once per message already in history at subscribe time (initial sync), and + * - once per new message pushed in real time thereafter, + * both in send order. + */ + subscribe(onMessage: (msg: ChatMessage) => void): Promise; + + /** Send a message as `sender`. Resolves once the backend accepts it. */ + send(sender: string, text: string): Promise; + + /** Tear down the connection. */ + close(): Promise; +} + +/** Each adapter module must default-export a factory that builds a fresh client. */ +export type ChatClientFactory = () => ChatClient; diff --git a/tools/stack-bench/_shared/harness/src/runScenario.ts b/tools/stack-bench/_shared/harness/src/runScenario.ts new file mode 100644 index 00000000000..5a618a92925 --- /dev/null +++ b/tools/stack-bench/_shared/harness/src/runScenario.ts @@ -0,0 +1,42 @@ +import { promises as fs } from "fs"; +import path from "path"; +import { runChatScenario } from "./scenario.js"; +import type { ChatClientFactory } from "./chatClient.js"; + +// Harbor's verification contract: write a reward to /logs/verifier/reward.txt. +// REWARD_DIR is overridable for local runs outside a Harbor container. +const REWARD_DIR = process.env.REWARD_DIR ?? "/logs/verifier"; + +async function writeReward(reward: number, extra: object = {}) { + await fs.mkdir(REWARD_DIR, { recursive: true }); + await fs.writeFile(path.join(REWARD_DIR, "reward.txt"), reward.toFixed(4) + "\n"); + await fs.writeFile(path.join(REWARD_DIR, "result.json"), JSON.stringify({ reward, ...extra }, null, 2)); +} + +async function main() { + // The per-backend adapter is selected by path: env ADAPTER_PATH or argv[2]. + const adapterPath = process.env.ADAPTER_PATH ?? process.argv[2]; + if (!adapterPath) throw new Error("ADAPTER_PATH (env) or argv path to the compiled adapter is required"); + + const mod = await import(path.resolve(adapterPath)); + const makeClient: ChatClientFactory = mod.default ?? mod.makeClient; + if (typeof makeClient !== "function") { + throw new Error(`adapter ${adapterPath} must default-export a ChatClientFactory`); + } + + const result = await runChatScenario(makeClient); + const reward = result.total === 0 ? 0 : result.passed / result.total; + await writeReward(reward, result); + + console.log(`\nstack-bench reward=${reward.toFixed(4)} (${result.passed}/${result.total} checks)`); + for (const c of result.checks) { + console.log(` [${c.pass ? "PASS" : "FAIL"}] ${c.name}${c.detail ? " — " + c.detail : ""}`); + } +} + +main().catch(async (err) => { + // Any harness/adapter crash is a 0 reward, not a Harbor infra failure. + console.error("stack-bench harness error:", err); + await writeReward(0, { error: String(err) }).catch(() => {}); + process.exit(0); +}); diff --git a/tools/stack-bench/_shared/harness/src/runTeamChat.ts b/tools/stack-bench/_shared/harness/src/runTeamChat.ts new file mode 100644 index 00000000000..693794598e7 --- /dev/null +++ b/tools/stack-bench/_shared/harness/src/runTeamChat.ts @@ -0,0 +1,68 @@ +// Entry point for the team-chat verifier. +// +// Harbor contract: write the scalar reward to /logs/verifier/reward.txt. We +// additionally write reward.json (multiple named metrics — Harbor supports +// float/int metric maps) and result.json (full per-check findings: the +// machine-readable payload a multi-step agent loop can feed back to the agent). +// +// Env: +// ADAPTER_PATH path to the per-backend adapter module (or argv[2]) +// RESTART_CMD shell command that restarts the backend process (durability +// phase). Typically "/opt/stack-bench/backendctl restart". +// REWARD_DIR override /logs/verifier for local runs. + +import { execSync } from "child_process"; +import { promises as fs } from "fs"; +import path from "path"; +import type { AppClientFactory } from "./appClient.js"; +import { runTeamChatScenario, scoreChecks, GROUP_WEIGHTS } from "./teamChat/scenario.js"; + +const REWARD_DIR = process.env.REWARD_DIR ?? "/logs/verifier"; + +async function writeOutputs(reward: number, rewardMetrics: Record, result: object) { + await fs.mkdir(REWARD_DIR, { recursive: true }); + await fs.writeFile(path.join(REWARD_DIR, "reward.txt"), reward.toFixed(4) + "\n"); + await fs.writeFile(path.join(REWARD_DIR, "reward.json"), JSON.stringify(rewardMetrics, null, 2)); + await fs.writeFile(path.join(REWARD_DIR, "result.json"), JSON.stringify(result, null, 2)); +} + +async function main() { + const adapterPath = process.env.ADAPTER_PATH ?? process.argv[2]; + if (!adapterPath) throw new Error("ADAPTER_PATH (env) or argv path to the adapter is required"); + + const mod = await import(path.resolve(adapterPath)); + const makeClient: AppClientFactory = mod.default ?? mod.makeClient; + if (typeof makeClient !== "function") { + throw new Error(`adapter ${adapterPath} must default-export an AppClientFactory`); + } + + const restartCmd = process.env.RESTART_CMD; + const restartBackend = restartCmd + ? async () => { + console.log(`\n== restarting backend: ${restartCmd}`); + execSync(restartCmd, { stdio: "inherit", timeout: 180_000 }); + } + : undefined; + + console.log("== team-chat scenario starting"); + const { checks, metrics } = await runTeamChatScenario({ makeClient, restartBackend }); + const { reward, groups } = scoreChecks(checks); + + const rewardMetrics: Record = { reward: Number(reward.toFixed(4)) }; + for (const [g, s] of Object.entries(groups)) rewardMetrics[g] = Number(s.score.toFixed(4)); + for (const [k, v] of Object.entries(metrics)) if (typeof v === "number" && isFinite(v)) rewardMetrics[k] = v; + + await writeOutputs(reward, rewardMetrics, { reward, weights: GROUP_WEIGHTS, groups, metrics, checks }); + + console.log(`\n== team-chat reward=${reward.toFixed(4)}`); + for (const [g, s] of Object.entries(groups)) { + console.log(` ${g.padEnd(12)} ${s.passed}/${s.total} (weight ${GROUP_WEIGHTS[g as keyof typeof GROUP_WEIGHTS]})`); + } +} + +main().catch(async (err) => { + // Any harness/adapter crash is a 0 reward, not a Harbor infra failure. + console.error("team-chat harness error:", err); + await writeOutputs(0, { reward: 0 }, { error: String(err) }).catch(() => {}); + process.exit(0); +}); diff --git a/tools/stack-bench/_shared/harness/src/scenario.ts b/tools/stack-bench/_shared/harness/src/scenario.ts new file mode 100644 index 00000000000..cdce2a6528e --- /dev/null +++ b/tools/stack-bench/_shared/harness/src/scenario.ts @@ -0,0 +1,96 @@ +import { ChatClient, ChatClientFactory, ChatMessage } from "./chatClient.js"; + +export interface CheckResult { + name: string; + pass: boolean; + detail?: string; +} + +export interface ScenarioResult { + passed: number; + total: number; + checks: CheckResult[]; +} + +const DELIVERY_TIMEOUT_MS = Number(process.env.DELIVERY_TIMEOUT_MS ?? 10_000); + +/** Poll `pred` until true or timeout. Returns whether it became true in time. */ +function waitFor(pred: () => boolean, timeoutMs: number, intervalMs = 100): Promise { + return new Promise((resolve) => { + const start = Date.now(); + const tick = () => { + if (pred()) return resolve(true); + if (Date.now() - start >= timeoutMs) return resolve(false); + setTimeout(tick, intervalMs); + }; + tick(); + }); +} + +/** + * The backend-neutral behavioral test for a minimal real-time chat app. + * + * Scoring is non-binary (Harbor supports fractional rewards): the reward is the + * fraction of checks that pass, so a backend that gets real-time delivery right + * but botches history still scores partial credit. + */ +export async function runChatScenario(makeClient: ChatClientFactory): Promise { + const checks: CheckResult[] = []; + const record = (name: string, pass: boolean, detail?: string) => checks.push({ name, pass, detail }); + + const a = makeClient(); + const b = makeClient(); + const bReceived: ChatMessage[] = []; + + try { + await a.connect(); + await b.connect(); + await b.subscribe((m) => bReceived.push(m)); + + // 1) Real-time delivery: B is subscribed; a message A sends must arrive (pushed). + await a.send("alice", "hello"); + const got1 = await waitFor(() => bReceived.some((m) => m.text === "hello"), DELIVERY_TIMEOUT_MS); + record("realtime_delivery", got1, got1 ? undefined : "B never received 'hello' within timeout"); + + // 2) Ordering: a second message arrives, after the first. + await a.send("alice", "world"); + const got2 = await waitFor( + () => bReceived.some((m) => m.text === "world"), + DELIVERY_TIMEOUT_MS, + ); + const texts = bReceived.map((m) => m.text); + const ordered = + texts.indexOf("hello") !== -1 && + texts.indexOf("world") !== -1 && + texts.indexOf("hello") < texts.indexOf("world"); + record("ordering", got2 && ordered, ordered ? undefined : `unexpected order: ${JSON.stringify(texts)}`); + + // 3) Sender attribution survives the round trip. + const helloMsg = bReceived.find((m) => m.text === "hello"); + record("sender_attribution", helloMsg?.sender === "alice", `sender=${helloMsg?.sender ?? ""}`); + + // 4) History persistence: a freshly-connected client must receive prior messages on subscribe. + const c = makeClient(); + const cReceived: ChatMessage[] = []; + await c.connect(); + await c.subscribe((m) => cReceived.push(m)); + const gotHistory = await waitFor( + () => cReceived.some((m) => m.text === "hello") && cReceived.some((m) => m.text === "world"), + DELIVERY_TIMEOUT_MS, + ); + record( + "history_persistence", + gotHistory, + gotHistory ? undefined : `fresh client saw: ${JSON.stringify(cReceived.map((m) => m.text))}`, + ); + await c.close(); + } catch (err) { + record("scenario_error", false, String(err)); + } finally { + await a.close().catch(() => {}); + await b.close().catch(() => {}); + } + + const passed = checks.filter((c) => c.pass).length; + return { passed, total: checks.length, checks }; +} diff --git a/tools/stack-bench/_shared/harness/src/teamChat/model.ts b/tools/stack-bench/_shared/harness/src/teamChat/model.ts new file mode 100644 index 00000000000..983f383ef93 --- /dev/null +++ b/tools/stack-bench/_shared/harness/src/teamChat/model.ts @@ -0,0 +1,134 @@ +// Expected-state model for the team-chat scenario (τ-bench pattern: the +// verifier is the only writer, so it can maintain the exact goal state and +// compare backend snapshots against it at quiescent points). +// +// The scenario applies an operation to the model ONLY when the backend +// accepted it (or was required to). Expected-failure operations never touch +// the model. + +import type { MemberRecord, MessageRecord, UserRecord } from "../appClient.js"; + +interface ModelMessage { + seq: number; + clientMsgId: string; + sender: string; + text: string; + edited: boolean; + deleted: boolean; +} + +interface ModelMember { + lastReadSeq: number; + unread: number; +} + +interface ModelRoom { + owner: string; + nextSeq: number; // seq the NEXT message will get + members: Map; + messages: ModelMessage[]; // seq order + seenClientMsgIds: Set; +} + +export class TeamChatModel { + users = new Map(); + rooms = new Map(); + + register(username: string) { + if (!this.users.has(username)) { + this.users.set(username, { status: "online", balance: 100 }); + } + } + + setStatus(username: string, status: string) { + this.users.get(username)!.status = status; + } + + createRoom(username: string, room: string) { + this.rooms.set(room, { + owner: username, + nextSeq: 1, + members: new Map([[username, { lastReadSeq: 0, unread: 0 }]]), + messages: [], + seenClientMsgIds: new Set(), + }); + } + + joinRoom(username: string, room: string) { + const r = this.rooms.get(room)!; + if (r.members.has(username)) return; + const unread = r.messages.filter((m) => m.sender !== username).length; + r.members.set(username, { lastReadSeq: 0, unread }); + } + + removeMember(username: string, room: string) { + this.rooms.get(room)!.members.delete(username); + } + + /** Returns the seq assigned, or null if deduped (idempotent resend). */ + sendMessage(sender: string, room: string, text: string, clientMsgId: string): number | null { + const r = this.rooms.get(room)!; + if (r.seenClientMsgIds.has(clientMsgId)) return null; + r.seenClientMsgIds.add(clientMsgId); + const seq = r.nextSeq++; + r.messages.push({ seq, clientMsgId, sender, text, edited: false, deleted: false }); + for (const [user, m] of r.members) { + if (user !== sender) m.unread += 1; + } + return seq; + } + + editMessage(room: string, clientMsgId: string, newText: string) { + const m = this.msg(room, clientMsgId); + m.text = newText; + m.edited = true; + } + + deleteMessage(room: string, clientMsgId: string) { + const m = this.msg(room, clientMsgId); + m.deleted = true; + m.text = ""; + } + + markRead(user: string, room: string, upToSeq: number) { + const r = this.rooms.get(room)!; + const mem = r.members.get(user)!; + mem.lastReadSeq = Math.max(mem.lastReadSeq, upToSeq); + mem.unread = r.messages.filter((m) => m.seq > mem.lastReadSeq && m.sender !== user).length; + } + + tip(fromUser: string, toUser: string, amount: number) { + this.users.get(fromUser)!.balance -= amount; + this.users.get(toUser)!.balance += amount; + } + + // ---- expected snapshots ---- + + expectedUser(username: string): UserRecord { + const u = this.users.get(username)!; + return { username, status: u.status, balance: u.balance }; + } + + expectedMembers(room: string): MemberRecord[] { + const r = this.rooms.get(room)!; + return [...r.members.entries()] + .map(([user, m]) => ({ user, lastReadSeq: m.lastReadSeq, unread: m.unread })) + .sort((a, b) => a.user.localeCompare(b.user)); + } + + expectedMessages(room: string): MessageRecord[] { + return this.rooms.get(room)!.messages.map((m) => ({ ...m })); + } + + nextSeq(room: string): number { + return this.rooms.get(room)!.nextSeq; + } + + totalBalance(...users: string[]): number { + return users.reduce((sum, u) => sum + this.users.get(u)!.balance, 0); + } + + private msg(room: string, clientMsgId: string): ModelMessage { + return this.rooms.get(room)!.messages.find((m) => m.clientMsgId === clientMsgId)!; + } +} diff --git a/tools/stack-bench/_shared/harness/src/teamChat/scenario.ts b/tools/stack-bench/_shared/harness/src/teamChat/scenario.ts new file mode 100644 index 00000000000..eac1e89518f --- /dev/null +++ b/tools/stack-bench/_shared/harness/src/teamChat/scenario.ts @@ -0,0 +1,663 @@ +// The backend-neutral behavioral scenario for the team-chat task. +// +// ~44 named checks in 4 weighted groups (correctness/realtime/durability/perf). +// The scenario is the only writer, so it maintains an exact expected-state +// model (τ-bench pattern) and compares backend snapshots against it at +// quiescent points. Concurrency checks (gapless seq, unread invariant, tip +// conservation) exercise transactional behavior; the restart phase exercises +// durability including durable counters and idempotency records. + +import type { AppClient, AppClientFactory, MessageRecord } from "../appClient.js"; +import { TeamChatModel } from "./model.js"; + +export type CheckGroup = "correctness" | "realtime" | "durability" | "perf"; + +export interface CheckResult { + group: CheckGroup; + name: string; + pass: boolean; + detail?: string; +} + +export interface ScenarioMetrics { + latency_p50_ms?: number; + latency_p95_ms?: number; + burst_throughput_msgs_per_sec?: number; +} + +export interface ScenarioResult { + checks: CheckResult[]; + metrics: ScenarioMetrics; +} + +export const GROUP_WEIGHTS: Record = { + correctness: 0.4, + realtime: 0.3, + durability: 0.2, + perf: 0.1, +}; + +const T = Number(process.env.DELIVERY_TIMEOUT_MS ?? 15_000); +const SETTLE_TIMEOUT_MS = Number(process.env.SETTLE_TIMEOUT_MS ?? 20_000); + +function waitFor(pred: () => boolean, timeoutMs = T, intervalMs = 100): Promise { + return new Promise((resolve) => { + const start = Date.now(); + const tick = () => { + if (pred()) return resolve(true); + if (Date.now() - start >= timeoutMs) return resolve(false); + setTimeout(tick, intervalMs); + }; + tick(); + }); +} + +/** Key-order-insensitive canonical serialization for deep equality. */ +function canon(x: unknown): string { + if (Array.isArray(x)) return "[" + x.map(canon).join(",") + "]"; + if (x !== null && typeof x === "object") { + const keys = Object.keys(x as object).sort(); + return "{" + keys.map((k) => JSON.stringify(k) + ":" + canon((x as any)[k])).join(",") + "}"; + } + return JSON.stringify(x) ?? "undefined"; +} + +/** Poll an async snapshot until it deep-equals `expected` (handles propagation lag). */ +async function settleEqual( + snap: () => Promise, + expected: X, + timeoutMs = SETTLE_TIMEOUT_MS, +): Promise<{ ok: boolean; last: X | undefined }> { + const want = canon(expected); + const start = Date.now(); + let last: X | undefined; + for (;;) { + try { + last = await snap(); + if (canon(last) === want) return { ok: true, last }; + } catch { + /* backend may still be settling/restarting */ + } + if (Date.now() - start >= timeoutMs) return { ok: false, last }; + await new Promise((r) => setTimeout(r, 250)); + } +} + +/** Did `op` reject? (expected-failure probe) */ +async function rejects(op: () => Promise): Promise { + try { + await op(); + return false; + } catch { + return true; + } +} + +const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)); +const percentile = (xs: number[], p: number) => { + const s = [...xs].sort((a, b) => a - b); + return s[Math.min(s.length - 1, Math.ceil((p / 100) * s.length) - 1)]; +}; + +/** Recorder for one room subscription: collects message events in arrival order. */ +class RoomRecorder { + events: MessageRecord[] = []; + /** newest event per clientMsgId (edits/deletes overwrite) */ + get latest(): Map { + const m = new Map(); + for (const e of this.events) m.set(e.clientMsgId, e); + return m; + } + seqsSeen(): number[] { + // first arrival per clientMsgId only (ignore edit/delete re-deliveries) + const seen = new Set(); + const out: number[] = []; + for (const e of this.events) { + if (!seen.has(e.clientMsgId)) { + seen.add(e.clientMsgId); + out.push(e.seq); + } + } + return out; + } + handlers() { + return { onMessage: (m: MessageRecord) => this.events.push(m) }; + } +} + +export interface ScenarioEnv { + makeClient: AppClientFactory; + /** Restart the backend process; resolves when the restart command exits. */ + restartBackend?: () => Promise; +} + +export async function runTeamChatScenario(env: ScenarioEnv): Promise { + const { makeClient } = env; + const checks: CheckResult[] = []; + const metrics: ScenarioMetrics = {}; + const rec = (group: CheckGroup, name: string, pass: boolean, detail?: string) => { + checks.push({ group, name, pass, detail: pass ? undefined : detail }); + console.log(` [${pass ? "PASS" : "FAIL"}] ${group}/${name}${!pass && detail ? " — " + detail : ""}`); + }; + + const model = new TeamChatModel(); + const open: AppClient[] = []; + const client = async () => { + const c = makeClient(); + await c.connect(); + open.push(c); + return c; + }; + + let driver!: AppClient; + try { + driver = await client(); + + // ========================================================================= + // CORRECTNESS + // ========================================================================= + for (const u of ["alice", "bob", "carol", "dave", "mallory"]) { + await driver.register(u); + model.register(u); + } + { + const a = await settleEqual(() => driver.getUser("alice"), model.expectedUser("alice")); + rec("correctness", "register_creates_user", a.ok, `got ${JSON.stringify(a.last)}`); + } + + await driver.setStatus("alice", "away"); + model.setStatus("alice", "away"); + await driver.register("alice"); // must be a no-op + { + const a = await settleEqual(() => driver.getUser("alice"), model.expectedUser("alice")); + rec("correctness", "register_idempotent_preserves_state", a.ok, `got ${JSON.stringify(a.last)}`); + } + { + const bad = await rejects(() => driver.setStatus("alice", "invisible")); + const unknown = await rejects(() => driver.setStatus("nobody", "online")); + rec("correctness", "set_status_validation", bad && unknown, `invalid-status rejected=${bad}, unknown-user rejected=${unknown}`); + } + + await driver.createRoom("alice", "general"); + model.createRoom("alice", "general"); + { + const owner = await driver.getRoomOwner("general"); + const mem = await settleEqual(() => driver.getMembers("general"), model.expectedMembers("general")); + rec("correctness", "create_room_owner_and_membership", owner === "alice" && mem.ok, `owner=${owner} members=${JSON.stringify(mem.last)}`); + } + rec("correctness", "duplicate_room_rejected", await rejects(() => driver.createRoom("bob", "general"))); + + for (const u of ["bob", "carol", "dave"]) { + await driver.joinRoom(u, "general"); + model.joinRoom(u, "general"); + } + await driver.leaveRoom("dave", "general"); + model.removeMember("dave", "general"); + { + const mem = await settleEqual(() => driver.getMembers("general"), model.expectedMembers("general")); + rec("correctness", "join_leave_membership", mem.ok, `members=${JSON.stringify(mem.last)}`); + } + + rec("correctness", "nonmember_send_rejected", await rejects(() => driver.sendMessage("mallory", "general", "hi", "mal-1"))); + rec("correctness", "empty_text_rejected", await rejects(() => driver.sendMessage("bob", "general", "", "bob-empty"))); + rec("correctness", "oversize_text_rejected", await rejects(() => driver.sendMessage("bob", "general", "x".repeat(4001), "bob-big"))); + rec("correctness", "unknown_room_send_rejected", await rejects(() => driver.sendMessage("bob", "nowhere", "hi", "bob-lost"))); + + await driver.sendMessage("bob", "general", "first!", "g-1"); + model.sendMessage("bob", "general", "first!", "g-1"); + await driver.sendMessage("bob", "general", "retry of first", "g-1"); // idempotent resend: must succeed, no dupe + { + const msgs = await settleEqual(() => driver.getMessages("general"), model.expectedMessages("general")); + rec("correctness", "idempotent_resend_no_duplicate", msgs.ok, `messages=${JSON.stringify(msgs.last)}`); + } + + for (let i = 2; i <= 5; i++) { + await driver.sendMessage("alice", "general", `serial ${i}`, `g-${i}`); + model.sendMessage("alice", "general", `serial ${i}`, `g-${i}`); + } + { + const msgs = await driver.getMessages("general"); + const seqs = msgs.map((m) => m.seq); + const ok = JSON.stringify(seqs) === JSON.stringify([1, 2, 3, 4, 5]); + rec("correctness", "seq_serial_gapless", ok, `seqs=${JSON.stringify(seqs)}`); + } + + // Concurrent sends from 3 separate connections: seq must stay gapless/unique. + const [cA, cB, cC] = [await client(), await client(), await client()]; + { + const jobs: Promise[] = []; + const senders: Array<[AppClient, string]> = [[cA, "alice"], [cB, "bob"], [cC, "carol"]]; + for (const [conn, user] of senders) { + for (let i = 0; i < 10; i++) { + jobs.push(conn.sendMessage(user, "general", `conc ${user} ${i}`, `conc-${user}-${i}`)); + } + } + const results = await Promise.allSettled(jobs); + const failed = results.filter((r) => r.status === "rejected").length; + // Wait until all 35 messages are visible, then apply the backend's chosen order. + const snapshot = await (async () => { + let msgs: MessageRecord[] = []; + const deadline = Date.now() + SETTLE_TIMEOUT_MS; + for (;;) { + msgs = await driver.getMessages("general"); + if (msgs.length >= 35 || Date.now() > deadline) break; + await sleep(250); + } + return { done: msgs.length >= 35, msgs }; + })(); + // Apply the 30 concurrent sends to the model in the order the backend chose. + for (const m of snapshot.msgs.filter((m) => m.clientMsgId.startsWith("conc-"))) { + model.sendMessage(m.sender, "general", m.text, m.clientMsgId); + } + const seqs = snapshot.msgs.map((m) => m.seq); + const expectSeqs = Array.from({ length: 35 }, (_, i) => i + 1); + const gapless = JSON.stringify(seqs) === JSON.stringify(expectSeqs); + rec( + "correctness", + "seq_concurrent_gapless", + failed === 0 && snapshot.done && gapless, + `failedSends=${failed} count=${snapshot.msgs.length} seqs=${JSON.stringify(seqs.slice(0, 40))}`, + ); + } + { + const mem = await settleEqual(() => driver.getMembers("general"), model.expectedMembers("general")); + rec("correctness", "unread_exact_after_concurrent_sends", mem.ok, `members=${JSON.stringify(mem.last)} expected=${JSON.stringify(model.expectedMembers("general"))}`); + } + { + await driver.markRead("carol", "general", 35); + model.markRead("carol", "general", 35); + const after = await settleEqual(() => driver.getMembers("general"), model.expectedMembers("general")); + await driver.markRead("carol", "general", 5); // must NOT regress + model.markRead("carol", "general", 5); + const mono = await settleEqual(() => driver.getMembers("general"), model.expectedMembers("general")); + rec("correctness", "mark_read_and_monotonicity", after.ok && mono.ok, `afterMark=${JSON.stringify(after.last)} afterRegress=${JSON.stringify(mono.last)}`); + } + + await driver.editMessage("bob", "general", "g-1", "first! (edited)"); + model.editMessage("general", "g-1", "first! (edited)"); + { + const msgs = await settleEqual(() => driver.getMessages("general"), model.expectedMessages("general")); + rec("correctness", "edit_by_author_applies", msgs.ok, `messages[0]=${JSON.stringify(msgs.last?.[0])}`); + } + rec("correctness", "edit_by_other_rejected", await rejects(() => driver.editMessage("carol", "general", "g-1", "hax"))); + + await driver.deleteMessage("alice", "general", "g-2"); // author deletes own message + model.deleteMessage("general", "g-2"); + await driver.deleteMessage("alice", "general", "g-1"); // room owner deletes bob's message + model.deleteMessage("general", "g-1"); + { + const msgs = await settleEqual(() => driver.getMessages("general"), model.expectedMessages("general")); + const g1 = msgs.last?.find((m) => m.clientMsgId === "g-1"); + rec("correctness", "delete_tombstone_by_owner_and_author", msgs.ok && g1?.deleted === true && g1?.text === "", `g-1=${JSON.stringify(g1)}`); + } + rec("correctness", "delete_by_other_rejected", await rejects(() => driver.deleteMessage("carol", "general", "g-3"))); + rec("correctness", "delete_already_deleted_rejected", await rejects(() => driver.deleteMessage("bob", "general", "g-1"))); + { + // Tombstone privacy: a completely fresh client must never see the original text. + const fresh = await client(); + const msgs = await fresh.getMessages("general"); + const g1 = msgs.find((m) => m.clientMsgId === "g-1"); + rec("correctness", "deleted_text_unrecoverable", g1 !== undefined && g1.deleted && g1.text === "", `freshView g-1=${JSON.stringify(g1)}`); + } + + // ---- tips ---- + await driver.tip("alice", "bob", 30); + model.tip("alice", "bob", 30); + { + const a = await settleEqual(() => driver.getUser("alice"), model.expectedUser("alice")); + const b = await settleEqual(() => driver.getUser("bob"), model.expectedUser("bob")); + rec("correctness", "tip_transfers_balance", a.ok && b.ok, `alice=${JSON.stringify(a.last)} bob=${JSON.stringify(b.last)}`); + } + rec("correctness", "tip_overdraft_rejected", await rejects(() => driver.tip("mallory", "alice", 5000))); + { + const zero = await rejects(() => driver.tip("alice", "bob", 0)); + const neg = await rejects(() => driver.tip("alice", "bob", -5)); + const self = await rejects(() => driver.tip("alice", "alice", 5)); + const unknown = await rejects(() => driver.tip("alice", "nobody", 5)); + rec("correctness", "tip_invalid_rejected", zero && neg && self && unknown, `zero=${zero} neg=${neg} self=${self} unknown=${unknown}`); + } + { + // Concurrent round-robin tips: all legal under any interleaving, so all + // must succeed and final balances are deterministic (net zero). + const before = model.totalBalance("alice", "bob", "carol"); + const jobs: Promise[] = []; + for (let round = 0; round < 3; round++) { + jobs.push(cA.tip("alice", "bob", 5)); + jobs.push(cB.tip("bob", "carol", 5)); + jobs.push(cC.tip("carol", "alice", 5)); + } + const results = await Promise.allSettled(jobs); + const failed = results.filter((r) => r.status === "rejected").length; + // net zero: model balances unchanged + const a = await settleEqual(() => driver.getUser("alice"), model.expectedUser("alice")); + const b = await settleEqual(() => driver.getUser("bob"), model.expectedUser("bob")); + const c = await settleEqual(() => driver.getUser("carol"), model.expectedUser("carol")); + const after = model.totalBalance("alice", "bob", "carol"); + rec( + "correctness", + "tip_concurrent_conservation", + failed === 0 && a.ok && b.ok && c.ok && before === after, + `failedTips=${failed} alice=${JSON.stringify(a.last)} bob=${JSON.stringify(b.last)} carol=${JSON.stringify(c.last)}`, + ); + } + + // ---- kick & permissions ---- + { + const nonOwnerKick = await rejects(() => driver.kick("bob", "general", "carol")); + const kickOwner = await rejects(() => driver.kick("bob", "general", "alice")); + const ownerLeave = await rejects(() => driver.leaveRoom("alice", "general")); + rec("correctness", "kick_and_leave_permissions", nonOwnerKick && kickOwner && ownerLeave, `nonOwnerKick=${nonOwnerKick} kickOwner=${kickOwner} ownerLeave=${ownerLeave}`); + } + { + await driver.kick("alice", "general", "carol"); + model.removeMember("carol", "general"); + const mem = await settleEqual(() => driver.getMembers("general"), model.expectedMembers("general")); + const blocked = await rejects(() => driver.sendMessage("carol", "general", "still here?", "carol-after-kick")); + rec("correctness", "kick_removes_membership_and_blocks_send", mem.ok && blocked, `members=${JSON.stringify(mem.last)} sendBlocked=${blocked}`); + } + + // ========================================================================= + // REALTIME — dedicated room "rt" with three independent subscriber clients + // ========================================================================= + await driver.createRoom("alice", "rt"); + model.createRoom("alice", "rt"); + for (const u of ["bob", "carol"]) { + await driver.joinRoom(u, "rt"); + model.joinRoom(u, "rt"); + } + + const obs1 = await client(); + const obs2 = await client(); + const obs3 = await client(); + const r1 = new RoomRecorder(); + const r2 = new RoomRecorder(); + const r3 = new RoomRecorder(); + await obs1.subscribeRoom("rt", r1.handlers()); + await obs2.subscribeRoom("rt", r2.handlers()); + await obs3.subscribeRoom("rt", r3.handlers()); + + // A general-room subscriber must not hear rt traffic (cross-room isolation). + const isoRec = new RoomRecorder(); + const isoClient = await client(); + await isoClient.subscribeRoom("general", isoRec.handlers()); + const generalCount = isoRec.events.length; // history size at subscribe + + { + const jobs: Promise[] = []; + const senders: Array<[AppClient, string]> = [[cA, "alice"], [cB, "bob"], [cC, "carol"]]; + for (const [conn, user] of senders) { + for (let i = 0; i < 5; i++) jobs.push(conn.sendMessage(user, "rt", `rt ${user} ${i}`, `rt-${user}-${i}`)); + } + await Promise.allSettled(jobs); + const allArrived = await waitFor( + () => [r1, r2, r3].every((r) => r.seqsSeen().length >= 15), + T, + ); + rec("realtime", "fanout_all_subscribers_receive", allArrived, `counts=${[r1, r2, r3].map((r) => r.seqsSeen().length).join(",")}`); + + // Sync the model with the backend's chosen order. + const msgs = await driver.getMessages("rt"); + for (const m of msgs) model.sendMessage(m.sender, "rt", m.text, m.clientMsgId); + + const exactlyOnceInOrder = [r1, r2, r3].every((r) => { + const seqs = r.seqsSeen(); + const uniq = new Set(seqs).size === seqs.length; + const ascending = seqs.every((s, i) => i === 0 || seqs[i - 1] < s); + return uniq && ascending && seqs.length === 15; + }); + rec("realtime", "fanout_exactly_once_in_seq_order", allArrived && exactlyOnceInOrder, `obs1Seqs=${JSON.stringify(r1.seqsSeen())}`); + } + { + await sleep(1000); // give any misrouted events time to arrive + const leaked = isoRec.events.slice(generalCount).filter((e) => e.clientMsgId.startsWith("rt-")); + rec("realtime", "cross_room_isolation", leaked.length === 0, `leaked=${JSON.stringify(leaked.map((e) => e.clientMsgId))}`); + } + { + // Late joiner: full history in seq order, then a live message, no dupes/gaps. + const late = await client(); + const lateRec = new RoomRecorder(); + await late.subscribeRoom("rt", lateRec.handlers()); + const historyArrived = await waitFor(() => lateRec.seqsSeen().length >= 15, T); + await driver.sendMessage("alice", "rt", "post-join live", "rt-live-1"); + model.sendMessage("alice", "rt", "post-join live", "rt-live-1"); + const liveArrived = await waitFor(() => lateRec.latest.has("rt-live-1"), T); + const seqs = lateRec.seqsSeen(); + const expected = Array.from({ length: 16 }, (_, i) => i + 1); + const exact = JSON.stringify(seqs) === JSON.stringify(expected); + rec("realtime", "late_joiner_history_then_live", historyArrived && liveArrived && exact, `seqs=${JSON.stringify(seqs)}`); + } + { + await driver.editMessage("alice", "rt", "rt-live-1", "post-join live (edited)"); + model.editMessage("rt", "rt-live-1", "post-join live (edited)"); + const saw = await waitFor(() => { + const m = r1.latest.get("rt-live-1"); + return m?.edited === true && m.text === "post-join live (edited)"; + }, T); + rec("realtime", "edit_propagates_live", saw, `obs1 sees ${JSON.stringify(r1.latest.get("rt-live-1"))}`); + } + { + await driver.deleteMessage("alice", "rt", "rt-live-1"); + model.deleteMessage("rt", "rt-live-1"); + const saw = await waitFor(() => { + const m = r1.latest.get("rt-live-1"); + return m?.deleted === true && m.text === ""; + }, T); + rec("realtime", "delete_propagates_live", saw, `obs1 sees ${JSON.stringify(r1.latest.get("rt-live-1"))}`); + } + { + const users = new Map(); + await obs1.subscribeUsers({ onUser: (u) => users.set(u.username, { status: u.status, balance: u.balance }) }); + await waitFor(() => users.has("bob"), T); + await driver.setStatus("bob", "offline"); + model.setStatus("bob", "offline"); + const saw = await waitFor(() => users.get("bob")?.status === "offline", T); + rec("realtime", "status_change_propagates", saw, `bob=${JSON.stringify(users.get("bob"))}`); + + const bobBefore = model.users.get("bob")!.balance; + await driver.tip("alice", "bob", 7); + model.tip("alice", "bob", 7); + const sawBal = await waitFor(() => users.get("bob")?.balance === bobBefore + 7, T); + rec("realtime", "balance_change_propagates", sawBal, `bob=${JSON.stringify(users.get("bob"))}`); + } + { + const members = new Map(); + const removed: string[] = []; + await obs2.subscribeMembers("rt", { + onMember: (m) => members.set(m.user, { lastReadSeq: m.lastReadSeq, unread: m.unread }), + onMemberRemoved: (u) => removed.push(u), + }); + await waitFor(() => members.has("carol"), T); + await driver.kick("alice", "rt", "carol"); + model.removeMember("carol", "rt"); + const saw = await waitFor(() => removed.includes("carol") || !members.has("carol"), T); + rec("realtime", "membership_change_propagates", saw, `removed=${JSON.stringify(removed)}`); + } + + // ========================================================================= + // PERF — dedicated room, generous thresholds; raw numbers reported as metrics + // ========================================================================= + await driver.createRoom("alice", "perf"); + model.createRoom("alice", "perf"); + for (const u of ["bob", "carol"]) { + await driver.joinRoom(u, "perf"); + model.joinRoom(u, "perf"); + } + const perfObs = await client(); + const perfRec = new RoomRecorder(); + const arrivalTimes = new Map(); + await perfObs.subscribeRoom("perf", { + onMessage: (m) => { + if (!arrivalTimes.has(m.clientMsgId)) arrivalTimes.set(m.clientMsgId, Date.now()); + perfRec.events.push(m); + }, + }); + { + const latencies: number[] = []; + for (let i = 0; i < 30; i++) { + const id = `lat-${i}`; + const t0 = Date.now(); + await cB.sendMessage("bob", "perf", `latency probe ${i}`, id); + model.sendMessage("bob", "perf", `latency probe ${i}`, id); + const got = await waitFor(() => arrivalTimes.has(id), T); + if (got) latencies.push(arrivalTimes.get(id)! - t0); + await sleep(100); + } + const p50 = latencies.length ? percentile(latencies, 50) : NaN; + const p95 = latencies.length ? percentile(latencies, 95) : NaN; + metrics.latency_p50_ms = p50; + metrics.latency_p95_ms = p95; + const ok = latencies.length === 30 && p95 < 1500; + rec("perf", "delivery_latency_p95", ok, `delivered=${latencies.length}/30 p50=${p50}ms p95=${p95}ms`); + } + { + const t0 = Date.now(); + const jobs: Promise[] = []; + const senders: Array<[AppClient, string]> = [[cA, "alice"], [cB, "bob"], [cC, "carol"]]; + for (const [conn, user] of senders) { + for (let i = 0; i < 40; i++) jobs.push(conn.sendMessage(user, "perf", `burst ${user} ${i}`, `burst-${user}-${i}`)); + } + const results = await Promise.allSettled(jobs); + const failed = results.filter((r) => r.status === "rejected").length; + const allArrived = await waitFor( + () => new Set(perfRec.events.filter((e) => e.clientMsgId.startsWith("burst-")).map((e) => e.clientMsgId)).size >= 120, + 45_000, + ); + const durS = (Date.now() - t0) / 1000; + metrics.burst_throughput_msgs_per_sec = allArrived ? Math.round((120 / durS) * 10) / 10 : 0; + // Sync model with backend order. + const msgs = await driver.getMessages("perf"); + for (const m of msgs) model.sendMessage(m.sender, "perf", m.text, m.clientMsgId); + rec("perf", "burst_delivery_within_budget", failed === 0 && allArrived, `failedSends=${failed} delivered in ${durS.toFixed(1)}s (${metrics.burst_throughput_msgs_per_sec} msg/s)`); + } + + // Final pre-restart settle + full goal-state comparison. + { + const parts = await Promise.all([ + settleEqual(() => driver.getMessages("general"), model.expectedMessages("general")), + settleEqual(() => driver.getMessages("rt"), model.expectedMessages("rt")), + settleEqual(() => driver.getMessages("perf"), model.expectedMessages("perf")), + settleEqual(() => driver.getMembers("general"), model.expectedMembers("general")), + settleEqual(() => driver.getMembers("rt"), model.expectedMembers("rt")), + settleEqual(() => driver.getMembers("perf"), model.expectedMembers("perf")), + settleEqual(() => driver.getUser("alice"), model.expectedUser("alice")), + settleEqual(() => driver.getUser("bob"), model.expectedUser("bob")), + settleEqual(() => driver.getUser("carol"), model.expectedUser("carol")), + ]); + const ok = parts.every((p) => p.ok); + rec("correctness", "final_state_matches_model", ok, `firstMismatch=${JSON.stringify(parts.find((p) => !p.ok)?.last)?.slice(0, 400)}`); + } + + // ========================================================================= + // DURABILITY — restart the backend, reconnect fresh, verify everything + // ========================================================================= + if (!env.restartBackend) { + for (const name of [ + "restart_users_and_balances_persist", + "restart_rooms_persist", + "restart_memberships_persist", + "restart_messages_persist", + "restart_seq_continues_gapless", + "restart_realtime_works", + "restart_idempotency_persists", + ]) { + rec("durability", name, false, "no RESTART_CMD configured — durability unverifiable"); + } + } else { + // Drop all live connections first; they die with the backend anyway. + for (const c of open.splice(0)) await c.close().catch(() => {}); + await env.restartBackend(); + + // Fresh client; connect may need retries while the backend comes back. + let post: AppClient | undefined; + const reconnectDeadline = Date.now() + 90_000; + let lastErr: unknown; + while (Date.now() < reconnectDeadline && !post) { + try { + const c = makeClient(); + await c.connect(); + await c.getUser("alice"); // probe a real read + post = c; + open.push(c); + } catch (e) { + lastErr = e; + await sleep(1000); + } + } + if (!post) throw new Error(`could not reconnect after restart: ${lastErr}`); + + { + const users = await Promise.all( + ["alice", "bob", "carol", "dave", "mallory"].map((u) => settleEqual(() => post!.getUser(u), model.expectedUser(u))), + ); + rec("durability", "restart_users_and_balances_persist", users.every((r) => r.ok), `mismatch=${JSON.stringify(users.find((r) => !r.ok)?.last)}`); + } + { + const owners = await Promise.all( + (["general", "rt", "perf"] as const).map(async (r) => (await post!.getRoomOwner(r)) === "alice"), + ); + rec("durability", "restart_rooms_persist", owners.every(Boolean), `owners ok=${JSON.stringify(owners)}`); + } + { + const mems = await Promise.all( + (["general", "rt", "perf"] as const).map((r) => settleEqual(() => post!.getMembers(r), model.expectedMembers(r))), + ); + rec("durability", "restart_memberships_persist", mems.every((m) => m.ok), `mismatch=${JSON.stringify(mems.find((m) => !m.ok)?.last)?.slice(0, 400)}`); + } + { + const msgs = await Promise.all( + (["general", "rt", "perf"] as const).map((r) => settleEqual(() => post!.getMessages(r), model.expectedMessages(r))), + ); + rec("durability", "restart_messages_persist", msgs.every((m) => m.ok), `mismatch=${JSON.stringify(msgs.find((m) => !m.ok)?.last)?.slice(0, 400)}`); + } + { + // Durable per-room counter: next seq continues exactly from pre-restart max. + const expectSeq = model.nextSeq("general"); + await post.sendMessage("alice", "general", "after restart", "g-post-restart"); + model.sendMessage("alice", "general", "after restart", "g-post-restart"); + const settled = await settleEqual(() => post!.getMessages("general"), model.expectedMessages("general")); + const got = settled.last?.find((m) => m.clientMsgId === "g-post-restart"); + rec("durability", "restart_seq_continues_gapless", settled.ok && got?.seq === expectSeq, `expected seq=${expectSeq} got=${JSON.stringify(got)}`); + } + { + const sub = await client(); + const subRec = new RoomRecorder(); + await sub.subscribeRoom("general", subRec.handlers()); + await post.sendMessage("bob", "general", "live after restart", "g-post-live"); + model.sendMessage("bob", "general", "live after restart", "g-post-live"); + const saw = await waitFor(() => subRec.latest.has("g-post-live"), T); + rec("durability", "restart_realtime_works", saw, "subscriber did not receive post-restart live message"); + } + { + // Idempotency record must be durable: pre-restart clientMsgId resend → no dupe. + await post.sendMessage("bob", "general", "dupe probe", "g-1"); + const settled = await settleEqual(() => post!.getMessages("general"), model.expectedMessages("general")); + rec("durability", "restart_idempotency_persists", settled.ok, `messages=${JSON.stringify(settled.last)?.slice(0, 400)}`); + } + } + } catch (err) { + checks.push({ group: "correctness", name: "scenario_error", pass: false, detail: String(err) }); + console.log(` [FAIL] correctness/scenario_error — ${String(err)}`); + } finally { + for (const c of open) await c.close().catch(() => {}); + } + + return { checks, metrics }; +} + +/** Weighted reward + per-group subscores from check results. */ +export function scoreChecks(checks: CheckResult[]): { + reward: number; + groups: Record; +} { + const groups = {} as Record; + for (const g of Object.keys(GROUP_WEIGHTS) as CheckGroup[]) { + const of = checks.filter((c) => c.group === g); + const passed = of.filter((c) => c.pass).length; + groups[g] = { passed, total: of.length, score: of.length ? passed / of.length : 0 }; + } + let reward = 0; + for (const g of Object.keys(GROUP_WEIGHTS) as CheckGroup[]) { + reward += GROUP_WEIGHTS[g] * groups[g].score; + } + return { reward, groups }; +} diff --git a/tools/stack-bench/_shared/harness/tsconfig.json b/tools/stack-bench/_shared/harness/tsconfig.json new file mode 100644 index 00000000000..399513e5ccb --- /dev/null +++ b/tools/stack-bench/_shared/harness/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true + }, + "include": ["src/**/*.ts"] +} diff --git a/tools/stack-bench/_shared/instruction.base.md b/tools/stack-bench/_shared/instruction.base.md new file mode 100644 index 00000000000..c7aa269ff6d --- /dev/null +++ b/tools/stack-bench/_shared/instruction.base.md @@ -0,0 +1,36 @@ + + +# Task: minimal real-time chat backend + +Build the backend for a minimal real-time chat application. There is a single +shared chat room. Clients connect, subscribe to the room, and send messages; +every connected client sees new messages pushed in real time, and a client that +connects later receives the full message history. + +## Functional requirements + +1. **Persisted messages.** Each message has a `sender` (string), a `text` + (string), and a server-assigned send time. Messages persist for the lifetime + of the deployment. +2. **Send.** A client can send a message by providing `sender` and `text`. + Reject empty `text`. +3. **Real-time subscription.** A subscribed client receives every message + currently in history at subscribe time, and every new message thereafter, + pushed (not polled), in send order. + +## What is graded + +An automated grader connects two clients via the backend's real-time client SDK +and checks: real-time delivery, message ordering, sender attribution, and +history persistence for a late-joining client. Grading is behavioral — it drives +the running app through its client SDK; it does not inspect your source. Partial +credit is awarded per check. + +Your implementation MUST expose exactly the identifiers named in the +**Contract** section below so the grader can connect. diff --git a/tools/stack-bench/_shared/team-chat.base.md b/tools/stack-bench/_shared/team-chat.base.md new file mode 100644 index 00000000000..5d0ab7bc0fd --- /dev/null +++ b/tools/stack-bench/_shared/team-chat.base.md @@ -0,0 +1,142 @@ + + +# Task: team chat backend + +Build the backend for a team chat application (think a minimal Slack/Discord +server): named rooms with owners and members, real-time messaging with +editing and deletion, per-member unread counters, user presence, and a credit +system where users can tip each other. + +The application is graded **behaviorally** by an automated verifier that +drives the running backend through its real client SDK with multiple +concurrent clients. It checks functional correctness, transactional +correctness under concurrency, real-time delivery, durability across a +backend restart, and basic performance. Partial credit is awarded per check. +Grading does not inspect your source code — only observable behavior counts. + +## Data model (required behavior, not storage advice) + +- **User** — `username` (unique string), `status` (one of `"online"`, + `"away"`, `"offline"`), `balance` (integer credits). +- **Room** — `name` (unique string), `owner` (a username). +- **Membership** — which users are in which rooms, plus per-member + `last_read_seq` and `unread` counters (see Unread rules). +- **Message** — belongs to a room; has a server-assigned per-room sequence + number `seq`, a client-supplied id `client_msg_id`, `sender`, `text`, + `edited` flag, `deleted` flag. + +## Operations + +All operations validate their inputs and **fail with an error** when a rule +below is violated. "Fails" must be observable to the caller through the +backend's normal error mechanism (rejected mutation / failed reducer call). + +1. **register(username)** — Creates the user with `status = "online"` and + `balance = 100`. If the user already exists, the call **succeeds** and + leaves the existing user completely unchanged (idempotent; must NOT reset + balance or status). +2. **set_status(username, status)** — Sets presence. Fails for unknown users + or a status outside the three allowed values. +3. **create_room(username, room)** — Creates a room owned by `username`, who + automatically becomes a member (with `last_read_seq = 0`, `unread = 0`). + Fails for unknown users or if the room name already exists. +4. **join_room(username, room)** — Adds the user as a member with + `last_read_seq = 0` and `unread` equal to the number of messages already + in the room sent by other users. Fails for unknown user/room. If already + a member, succeeds without changing the existing membership state. +5. **leave_room(username, room)** — Removes the membership. Fails for + unknown user/room, if not a member, or if the user is the room's owner + (owners cannot leave). +6. **kick(actor, room, target)** — Removes `target`'s membership. Fails + unless `actor` is the room's owner; fails if `target` is the owner or not + a member. +7. **send_message(sender, room, text, client_msg_id)** — Appends a message. + Fails for: unknown user/room, sender not a member, empty `text`, `text` + longer than 4000 characters. **Idempotency:** if a message with the same + `client_msg_id` already exists in the room, the call **succeeds** without + creating a second message (safe retry). Otherwise the server assigns + `seq` (see Sequence rules), stores the message with `edited = false`, + `deleted = false`, and **atomically** increments `unread` by 1 for every + member of the room except the sender. +8. **edit_message(actor, room, client_msg_id, new_text)** — Replaces the + text and sets `edited = true`. Fails unless `actor` is the original + sender; fails if the message is deleted or `new_text` fails the same + validation as send. +9. **delete_message(actor, room, client_msg_id)** — Tombstones the message: + sets `deleted = true` and **clears `text` to the empty string**. Allowed + for the original sender or the room owner; fails for anyone else or if + already deleted. The original text must not be recoverable by any client + afterwards (late-joining clients must see the tombstone, never the text). +10. **mark_read(user, room, up_to_seq)** — Sets the member's + `last_read_seq = max(current, up_to_seq)` (it never decreases), then + recomputes `unread` per the Unread rules. Fails if not a member. +11. **tip(from_user, to_user, amount)** — Atomically transfers `amount` + credits. Fails for: unknown users, `from_user == to_user`, + `amount <= 0`, or `from_user`'s balance below `amount`. Balances must + never go negative and total credits must be conserved, including under + concurrent tips. + +## Sequence rules (transactional correctness) + +- `seq` is **per room**, assigned by the server, starting at 1 for the + room's first message and increasing by exactly 1 per stored message — + **no gaps, no duplicates**, even when many clients send concurrently. +- The counter must survive a backend restart: the first message stored + after a restart continues from the pre-restart maximum (durable counter — + an in-memory counter that resets and reuses sequence numbers fails). + +## Unread rules + +At any quiescent moment, for every member `u` of room `r`: +`unread(u, r) == count of messages m in r with m.seq > last_read_seq(u, r) and m.sender != u`. +Deleted (tombstoned) messages still count — they were sent. This invariant +must hold exactly, including after concurrent sends from multiple clients +(the per-message unread increments must be atomic with message insertion). + +## Real-time requirements + +Connected clients observe changes via **push** (the backend's real-time +subscription mechanism, not client polling): + +- A client subscribed to a room receives every existing message at + subscribe time (in `seq` order, reflecting current edited/deleted state) + and every subsequent message thereafter, exactly once, in `seq` order. +- Message **edits and deletions** are pushed to subscribers in real time. +- Membership changes (join / leave / kick / read-state updates) are pushed + to subscribers of the room's membership. +- User changes (status, balance) are pushed to subscribers of the user list. +- Subscriptions are **per room**: a client subscribed to room A must not + receive room B's messages. + +## Durability requirements + +All state — users, balances, rooms, memberships, read state, messages +(including edits and tombstones), the idempotency record of seen +`client_msg_id`s, and the per-room sequence counters — must survive a +backend process restart. The verifier will restart the backend mid-run, +reconnect fresh clients, and check every piece of state plus continued +real-time operation. + +## Performance requirements + +Modest but real: with a handful of clients, message delivery latency +(send-call to another subscribed client receiving the push) should be well +under 1.5 seconds at p95, and a burst of 120 messages from 3 concurrent +senders should be fully delivered to a subscriber within 45 seconds. A +correct implementation on this backend passes these comfortably; polling +loops with long intervals do not. + +## What is graded + +The verifier awards partial credit across four weighted groups: +correctness & transactions (0.40), real-time behavior (0.30), durability +across restart (0.20), performance (0.10). Every check is machine-verified +through the client SDK against the running backend. + +Your implementation MUST expose exactly the identifiers named in the +**Contract** section below so the grader can connect. diff --git a/tools/stack-bench/dataset/README.md b/tools/stack-bench/dataset/README.md new file mode 100644 index 00000000000..1efb52d5bb0 --- /dev/null +++ b/tools/stack-bench/dataset/README.md @@ -0,0 +1 @@ +# clockwork/stack-bench diff --git a/tools/stack-bench/dataset/dataset.toml b/tools/stack-bench/dataset/dataset.toml new file mode 100644 index 00000000000..9d7aaf1d4c7 --- /dev/null +++ b/tools/stack-bench/dataset/dataset.toml @@ -0,0 +1,21 @@ +# Dataset manifest for clockwork/stack-bench +# Add tasks using: harbor add / +# Publish using: harbor publish + +[dataset] +name = "clockwork/stack-bench" +description = "stack-bench: build a minimal real-time chat backend; rank backends (SpacetimeDB vs Convex vs …) by how well an AI agent builds on each." +keywords = [] +[[dataset.authors]] +name = "Clockwork Labs" +email = "tyler@clockworklabs.io" + + +[[tasks]] +name = "clockwork/realtime-chat-spacetimedb" +digest = "sha256:fc9d05f03fb61414635133bbcd4c9fd4c0591853390f636d1775aa17a7c50a29" + +[[tasks]] +name = "clockwork/realtime-chat-convex" +digest = "sha256:531c8d6d4b147ef78f0295c3eda4ea4c8d20539823f0ba9a97744f675c71c5f8" + diff --git a/tools/stack-bench/tasks/realtime-chat/convex/environment/Dockerfile b/tools/stack-bench/tasks/realtime-chat/convex/environment/Dockerfile new file mode 100644 index 00000000000..e504f3ddbf1 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/environment/Dockerfile @@ -0,0 +1,9 @@ +# The agent's container ("main"). Node 22 (global WebSocket for the Convex client); +# `npx convex` is used at deploy time. The Convex backend itself runs as a separate +# compose service (see docker-compose.yaml). +FROM node:22-bookworm + +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app diff --git a/tools/stack-bench/tasks/realtime-chat/convex/environment/docker-compose.yaml b/tools/stack-bench/tasks/realtime-chat/convex/environment/docker-compose.yaml new file mode 100644 index 00000000000..d502a27f340 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/environment/docker-compose.yaml @@ -0,0 +1,21 @@ +# Overlaid onto Harbor's base compose. Adds a self-hosted Convex backend service +# alongside Harbor's `main` (agent) service. Both share the compose network, so +# `main` (where the agent works and the SHARED-mode verifier runs) reaches the +# backend at http://convex-backend:3210. +services: + convex-backend: + image: ghcr.io/get-convex/convex-backend:latest + environment: + # Fixed name+secret → deterministic admin key (see solution/solve.sh), so the + # benchmark is hermetic and reproducible. + - INSTANCE_NAME=convex-stack-bench + - INSTANCE_SECRET=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef + # Advertise the in-network origin so client WS URLs resolve to the service name. + - CONVEX_CLOUD_ORIGIN=http://convex-backend:3210 + - CONVEX_SITE_ORIGIN=http://convex-backend:3211 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3210/version"] + interval: 5s + timeout: 5s + start_period: 5s + retries: 20 diff --git a/tools/stack-bench/tasks/realtime-chat/convex/instruction.md b/tools/stack-bench/tasks/realtime-chat/convex/instruction.md new file mode 100644 index 00000000000..62e8e1e5b28 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/instruction.md @@ -0,0 +1,45 @@ +# Task: minimal real-time chat backend + +Build the backend for a minimal real-time chat application. There is a single +shared chat room. Clients connect, subscribe to the room, and send messages; +every connected client sees new messages pushed in real time, and a client that +connects later receives the full message history. + +## Functional requirements + +1. **Persisted messages.** Each message has a `sender` (string), a `text` + (string), and a server-assigned send time. Messages persist for the lifetime + of the deployment. +2. **Send.** A client can send a message by providing `sender` and `text`. + Reject empty `text`. +3. **Real-time subscription.** A subscribed client receives every message + currently in history at subscribe time, and every new message thereafter, + pushed (not polled), in send order. + +## What is graded + +An automated grader connects two clients via the Convex client SDK and checks: +real-time delivery, message ordering, sender attribution, and history +persistence for a late-joining client. Grading is behavioral — it drives the +running app through the SDK; it does not inspect your source. Partial credit is +awarded per check. + +## Contract (Convex) + +Build a Convex app and deploy it so the grader can connect via `CONVEX_URL` +(set in this environment). + +- **Table `messages`** with fields, named exactly: `sender` (string), + `text` (string), `sentAt` (number, ms since epoch). +- **Mutation `messages:sendMessage`** taking `{ sender: string, text: string }`, + inserting one `messages` row with `sentAt` set to the current time. It must + throw for empty `text`. +- **Query `messages:listMessages`** taking no args, returning all messages in + ascending send order. This query must be reactive (the grader subscribes via + `client.onUpdate(api.messages.listMessages, {})`). + +The exact `messages` table, `sendMessage`, and `listMessages` names are required +— the grader depends on them. + +You have 1800 seconds to complete this task. Do not cheat by using online +solutions or hints specific to this task. diff --git a/tools/stack-bench/tasks/realtime-chat/convex/solution/app/convex/messages.ts b/tools/stack-bench/tasks/realtime-chat/convex/solution/app/convex/messages.ts new file mode 100644 index 00000000000..4c161e43ed9 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/solution/app/convex/messages.ts @@ -0,0 +1,22 @@ +// Reference (oracle) Convex functions for the stack-bench realtime-chat task. +import { query, mutation } from "./_generated/server"; +import { v } from "convex/values"; + +export const listMessages = query({ + args: {}, + handler: async (ctx) => { + // Reactive by construction: clients subscribed via onUpdate re-receive this + // on every write to `messages`. Ascending insertion order. + return await ctx.db.query("messages").collect(); + }, +}); + +export const sendMessage = mutation({ + args: { sender: v.string(), text: v.string() }, + handler: async (ctx, { sender, text }) => { + if (text.length === 0) { + throw new Error("text must not be empty"); + } + await ctx.db.insert("messages", { sender, text, sentAt: Date.now() }); + }, +}); diff --git a/tools/stack-bench/tasks/realtime-chat/convex/solution/app/convex/schema.ts b/tools/stack-bench/tasks/realtime-chat/convex/solution/app/convex/schema.ts new file mode 100644 index 00000000000..849ff2af274 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/solution/app/convex/schema.ts @@ -0,0 +1,11 @@ +// Reference (oracle) Convex schema for the stack-bench realtime-chat task. +import { defineSchema, defineTable } from "convex/server"; +import { v } from "convex/values"; + +export default defineSchema({ + messages: defineTable({ + sender: v.string(), + text: v.string(), + sentAt: v.number(), + }), +}); diff --git a/tools/stack-bench/tasks/realtime-chat/convex/solution/app/package.json b/tools/stack-bench/tasks/realtime-chat/convex/solution/app/package.json new file mode 100644 index 00000000000..35b883e539f --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/solution/app/package.json @@ -0,0 +1,9 @@ +{ + "name": "chat-convex-oracle", + "version": "0.1.0", + "private": true, + "type": "module", + "dependencies": { + "convex": "^1.17.0" + } +} diff --git a/tools/stack-bench/tasks/realtime-chat/convex/solution/solve.sh b/tools/stack-bench/tasks/realtime-chat/convex/solution/solve.sh new file mode 100755 index 00000000000..0c7a0fd9a7c --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/solution/solve.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Thin launcher — Harbor's oracle entry point. Deploys the reference Convex app to +# the self-hosted backend service. Readiness is gated by the task.toml healthcheck. +set -euo pipefail +cd "$(dirname "$0")/app" +npm install --no-audit --no-fund --silent +export CONVEX_SELF_HOSTED_URL="${CONVEX_SELF_HOSTED_URL:-http://convex-backend:3210}" +# Deterministic admin key for the fixed INSTANCE_NAME/SECRET in environment/docker-compose.yaml. +export CONVEX_SELF_HOSTED_ADMIN_KEY="${CONVEX_SELF_HOSTED_ADMIN_KEY:-convex-stack-bench|01be067fd1488e360c17a915fd342953e6450766d4831138261df4371e27342009f370391e}" +exec npx convex deploy diff --git a/tools/stack-bench/tasks/realtime-chat/convex/task.toml b/tools/stack-bench/tasks/realtime-chat/convex/task.toml new file mode 100644 index 00000000000..b407e946287 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/task.toml @@ -0,0 +1,42 @@ +schema_version = "1.2" +artifacts = [] + +[task] +name = "clockwork/realtime-chat-convex" +description = "Build a minimal real-time chat backend on Convex; graded behaviorally through the Convex client SDK." +keywords = ["realtime", "chat", "convex", "cross-backend", "stack-bench"] +[[task.authors]] +name = "Clockwork Labs" +email = "" + +[metadata] +backend = "convex" +# Grouping dimensions read by `harbor stats breakdown` (tags/category/difficulty). +tags = ["convex"] +category = "realtime-chat" + +[verifier] +timeout_sec = 600.0 +# SHARED mode (default): verifier runs in `main` and reaches the convex-backend +# service over the compose network. Do NOT add [verifier.environment]. + +[agent] +timeout_sec = 1800.0 + +[environment] +build_timeout_sec = 1200.0 +os = "linux" +cpus = 2 +memory_mb = 4096 +storage_mb = 12288 +gpus = 0 +allow_internet = true + +# Wait for the convex-backend compose service to be reachable from `main` before +# the agent runs. +[environment.healthcheck] +command = "curl -sf -o /dev/null http://convex-backend:3210/version" +interval_sec = 3.0 +timeout_sec = 10.0 +start_period_sec = 5.0 +retries = 40 diff --git a/tools/stack-bench/tasks/realtime-chat/convex/tests/Dockerfile b/tools/stack-bench/tasks/realtime-chat/convex/tests/Dockerfile new file mode 100644 index 00000000000..1726f392a75 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/tests/Dockerfile @@ -0,0 +1,10 @@ +# Verifier container (run separately from the environment, per TB3). +FROM node:20-bookworm + +WORKDIR /tests +COPY . /tests/ + +# TODO(spike): pre-install + pre-build deps here so the verifier needs no +# network at grade time. The spike installs at run time for faster iteration. + +CMD ["bash", "test.sh"] diff --git a/tools/stack-bench/tasks/realtime-chat/convex/tests/adapter.ts b/tools/stack-bench/tasks/realtime-chat/convex/tests/adapter.ts new file mode 100644 index 00000000000..658092025d9 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/tests/adapter.ts @@ -0,0 +1,56 @@ +// Convex adapter: implements the stack-bench ChatClient against the running app. +// +// VALIDATED against self-hosted convex-backend (convex npm SDK). Uses `anyApi` so +// the grader references functions by name (messages.*) without importing the app's +// generated bindings. Run via tsx; Node 22+ supplies the global WebSocket. +// +// In Harbor SHARED mode the grader runs in `main` and reaches the backend service +// at http://convex-backend:3210 over the compose network. + +import { ConvexClient } from "convex/browser"; +import { anyApi } from "convex/server"; +import type { ChatClient, ChatMessage } from "./harness/src/chatClient.js"; + +const CONVEX_URL = process.env.CONVEX_URL ?? "http://convex-backend:3210"; + +class ConvexChatClient implements ChatClient { + private client: ConvexClient; + private unsubscribe?: () => void; + + constructor(url: string) { + this.client = new ConvexClient(url); + } + + async connect(): Promise { + // ConvexClient connects lazily on the first query/mutation. + } + + async subscribe(onMessage: (msg: ChatMessage) => void): Promise { + let seen = 0; + // onUpdate delivers the full list immediately (history) and on every change + // (real-time). Diff against `seen` to emit each message once, in order. + this.unsubscribe = this.client.onUpdate( + anyApi.messages.listMessages, + {}, + (rows: Array<{ sender: string; text: string }>) => { + for (let i = seen; i < rows.length; i++) { + onMessage({ sender: rows[i].sender, text: rows[i].text }); + } + seen = rows.length; + }, + ); + } + + async send(sender: string, text: string): Promise { + await this.client.mutation(anyApi.messages.sendMessage, { sender, text }); + } + + async close(): Promise { + this.unsubscribe?.(); + await this.client.close(); + } +} + +export default function makeClient(): ChatClient { + return new ConvexChatClient(CONVEX_URL); +} diff --git a/tools/stack-bench/tasks/realtime-chat/convex/tests/package.json b/tools/stack-bench/tasks/realtime-chat/convex/tests/package.json new file mode 100644 index 00000000000..272306e3659 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/tests/package.json @@ -0,0 +1,13 @@ +{ + "name": "stack-bench-verifier-convex", + "version": "0.0.1", + "private": true, + "type": "module", + "description": "Convex verifier: runs the shared behavioral scenario via the Convex client SDK.", + "dependencies": { + "convex": "^1.17.0" + }, + "devDependencies": { + "tsx": "^4.19.0" + } +} diff --git a/tools/stack-bench/tasks/realtime-chat/convex/tests/test.sh b/tools/stack-bench/tasks/realtime-chat/convex/tests/test.sh new file mode 100755 index 00000000000..badd1b432d6 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/convex/tests/test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Thin launcher — Harbor's verifier entry point. Runs the shared behavioral scenario +# through the Convex adapter; the harness writes /logs/verifier/reward.txt. Connects +# to the convex-backend compose service (SHARED mode). +set -euo pipefail +cd "$(dirname "$0")" +npm install --no-audit --no-fund --silent +exec npx tsx harness/src/runScenario.ts "$(pwd)/adapter.ts" diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/environment/Dockerfile b/tools/stack-bench/tasks/realtime-chat/spacetimedb/environment/Dockerfile new file mode 100644 index 00000000000..dc060a29c6f --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/environment/Dockerfile @@ -0,0 +1,32 @@ +# Environment the agent works in: Rust + SpacetimeDB CLI + Node 22, with a local +# SpacetimeDB instance started in the background by the entrypoint (so it's up for +# the agent and stays up for the SHARED-mode verifier). +# Rust 1.90: the spacetimedb crates require the `edition2024` Cargo feature +# (Rust >= 1.85, some deps want >= 1.87), so 1.83 is too old. +FROM rust:1.90-bookworm + +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +# Node 22 — supplies the global WebSocket the SpacetimeDB TS SDK uses. +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +# wasm target for building SpacetimeDB modules. +RUN rustup target add wasm32-unknown-unknown + +# SpacetimeDB CLI. TODO(spike): pin to 2.2.0 to match the committed bindings/SDK +# if the latest installer drifts and the WS protocol changes. +RUN curl -sSf https://install.spacetimedb.com | sh -s -- --yes +ENV PATH="/root/.local/bin:/root/.local/share/spacetime/bin/current:${PATH}" + +RUN mkdir -p /var/lib/stdb +WORKDIR /app +EXPOSE 3000 + +# Background a standalone SpacetimeDB instance, then exec Harbor's keepalive command +# (sleep infinity, passed as args) so the agent and the SHARED-mode verifier share +# this one container with the server running. +ENTRYPOINT ["/bin/sh", "-c", "spacetime start --data-dir /var/lib/stdb --listen-addr 0.0.0.0:3000 --non-interactive >/var/log/spacetime.log 2>&1 & exec \"$@\"", "sh"] +CMD ["sh", "-c", "sleep infinity"] diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/instruction.md b/tools/stack-bench/tasks/realtime-chat/spacetimedb/instruction.md new file mode 100644 index 00000000000..2dd9561ad43 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/instruction.md @@ -0,0 +1,51 @@ +# Task: minimal real-time chat backend + +Build the backend for a minimal real-time chat application. There is a single +shared chat room. Clients connect, subscribe to the room, and send messages; +every connected client sees new messages pushed in real time, and a client that +connects later receives the full message history. + +## Functional requirements + +1. **Persisted messages.** Each message has a `sender` (string), a `text` + (string), and a server-assigned send time. Messages persist for the lifetime + of the deployment. +2. **Send.** A client can send a message by providing `sender` and `text`. + Reject empty `text`. +3. **Real-time subscription.** A subscribed client receives every message + currently in history at subscribe time, and every new message thereafter, + pushed (not polled), in send order. + +## What is graded + +An automated grader connects two clients via the SpacetimeDB TypeScript SDK and +checks: real-time delivery, message ordering, sender attribution, and history +persistence for a late-joining client. Grading is behavioral — it drives the +running app through the SDK; it does not inspect your source. Partial credit is +awarded per check. + +## Contract (SpacetimeDB) + +Implement a SpacetimeDB module and publish it to the local instance so the +grader can connect to it. + +- **Database name:** `chat`, published to the `local` server + (`spacetime publish --server local chat`). A local SpacetimeDB instance is + already running in this environment. +- **Public table `message`** with columns, named exactly: + - `id`: `u64`, primary key, auto-increment + - `sender`: `String` + - `text`: `String` + - `sent_at`: `Timestamp` +- **Reducer `send_message(sender: String, text: String)`** that inserts one + `message` row with the given `sender` and `text`, `sent_at` set to the current + reducer timestamp, and an auto-assigned `id`. It must return an error for + empty `text`. +- Clients subscribe with `SELECT * FROM message`. + +You may use Rust, C#, or TypeScript for the module. The reference solution is in +Rust. The exact table/column/reducer names above are required — the grader +depends on them. + +You have 1800 seconds to complete this task. Do not cheat by using online +solutions or hints specific to this task. diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/solution/module/Cargo.toml b/tools/stack-bench/tasks/realtime-chat/spacetimedb/solution/module/Cargo.toml new file mode 100644 index 00000000000..6fdf019448a --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/solution/module/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "chat-module" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +# Pin to the SpacetimeDB version installed in the environment image. +spacetimedb = "1" diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/solution/module/src/lib.rs b/tools/stack-bench/tasks/realtime-chat/spacetimedb/solution/module/src/lib.rs new file mode 100644 index 00000000000..ac8d4c3a69a --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/solution/module/src/lib.rs @@ -0,0 +1,31 @@ +// Reference (oracle) SpacetimeDB chat module for the stack-bench realtime-chat task. +// +// NOTE: written against the SpacetimeDB 1.x bindings. Verify it builds against +// the `spacetime` version pinned in ../../environment/Dockerfile before relying +// on `harbor run -a oracle` scoring 1.0. + +use spacetimedb::{reducer, table, ReducerContext, Table, Timestamp}; + +#[table(name = message, public)] +pub struct Message { + #[primary_key] + #[auto_inc] + pub id: u64, + pub sender: String, + pub text: String, + pub sent_at: Timestamp, +} + +#[reducer] +pub fn send_message(ctx: &ReducerContext, sender: String, text: String) -> Result<(), String> { + if text.is_empty() { + return Err("text must not be empty".to_string()); + } + ctx.db.message().insert(Message { + id: 0, // auto_inc: 0 is a placeholder; the real id is assigned on insert + sender, + text, + sent_at: ctx.timestamp, + }); + Ok(()) +} diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/solution/solve.sh b/tools/stack-bench/tasks/realtime-chat/spacetimedb/solution/solve.sh new file mode 100755 index 00000000000..ccdd48776e7 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/solution/solve.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# Thin launcher — Harbor's oracle entry point (Harbor requires solve.sh on Linux). +# Publishes the reference module under ./module. Backend readiness is gated by the +# task.toml healthcheck, so no wait is needed here. +set -euo pipefail +exec spacetime publish --server "${STDB_SERVER:-http://127.0.0.1:3000}" -y \ + -p "$(dirname "$0")/module" chat diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/task.toml b/tools/stack-bench/tasks/realtime-chat/spacetimedb/task.toml new file mode 100644 index 00000000000..c1e624c1419 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/task.toml @@ -0,0 +1,48 @@ +schema_version = "1.2" +artifacts = [] + +[task] +name = "clockwork/realtime-chat-spacetimedb" +description = "Build a minimal real-time chat backend on SpacetimeDB; graded behaviorally through the SpacetimeDB TypeScript SDK." +keywords = ["realtime", "chat", "spacetimedb", "cross-backend", "stack-bench"] +[[task.authors]] +name = "Clockwork Labs" +email = "" + +[metadata] +# stack-bench-specific: which backend this task variant exercises. The same +# behavioral grader runs for every backend; only this value + the environment differ. +backend = "spacetimedb" +# Grouping dimensions read by `harbor stats breakdown` (tags/category/difficulty). +# The backend is the comparison axis, so it goes in tags. +tags = ["spacetimedb"] +category = "realtime-chat" + +[verifier] +timeout_sec = 600.0 +# SHARED mode (the default when no [verifier.environment] is set): the verifier +# runs INSIDE the agent's container, so it reaches the running app on localhost. +# Do NOT add a [verifier.environment] — that forces SEPARATE mode, an isolated +# compose project with no network path to the app. + +[agent] +timeout_sec = 1800.0 + +[environment] +build_timeout_sec = 1800.0 +os = "linux" +cpus = 2 +memory_mb = 4096 +storage_mb = 12288 +gpus = 0 +allow_internet = true + +# Wait for the in-container SpacetimeDB instance (started by the Dockerfile +# entrypoint) to accept connections before the agent runs. The server returns +# HTTP 404 on `/`, so a plain (non -f) curl that just confirms the port answers. +[environment.healthcheck] +command = "curl -s -o /dev/null http://127.0.0.1:3000" +interval_sec = 3.0 +timeout_sec = 10.0 +start_period_sec = 5.0 +retries = 30 diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/adapter.ts b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/adapter.ts new file mode 100644 index 00000000000..6d907d5c85b --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/adapter.ts @@ -0,0 +1,65 @@ +// SpacetimeDB adapter: implements the stack-bench ChatClient against the running +// `chat` database using the SpacetimeDB TypeScript SDK + generated bindings. +// +// VALIDATED against spacetimedb CLI/SDK 2.2.0 (see module_bindings/, pinned in +// package.json). Run via `tsx` — the generated bindings use extensionless ESM +// imports that tsc+node (NodeNext) reject but esbuild/tsx resolve. Node 22+ +// supplies the global WebSocket the SDK needs. +// +// In Harbor SHARED verifier mode the grader runs in the agent's container, so we +// connect to the in-container instance on localhost. + +import { DbConnection } from "./module_bindings/index.js"; +import type { ChatClient, ChatMessage } from "./harness/src/chatClient.js"; + +const URI = process.env.STDB_URI ?? "ws://127.0.0.1:3000"; +const DB = process.env.STDB_DB ?? "chat"; + +class SpacetimeChatClient implements ChatClient { + private conn: any; + private connected!: Promise; + + async connect(): Promise { + this.connected = new Promise((resolve, reject) => { + this.conn = DbConnection.builder() + .withUri(URI) + .withDatabaseName(DB) + .onConnect(() => resolve()) + .onConnectError((_ctx: any, err: any) => reject(err)) + .build(); + }); + await this.connected; + } + + async subscribe(onMessage: (msg: ChatMessage) => void): Promise { + // Register the row callback BEFORE subscribing so initial-sync (history) rows + // are delivered too — onInsert fires for both initial sync and live inserts. + this.conn.db.message.onInsert((_ctx: any, row: any) => { + onMessage({ sender: row.sender, text: row.text }); + }); + await new Promise((resolve, reject) => { + this.conn + .subscriptionBuilder() + .onApplied(() => resolve()) + .onError((ctx: any) => reject(new Error("subscription error: " + String(ctx)))) + .subscribe(["SELECT * FROM message"]); + }); + } + + async send(sender: string, text: string): Promise { + // Reducer takes a single object arg, not positional. + this.conn.reducers.sendMessage({ sender, text }); + } + + async close(): Promise { + try { + this.conn?.disconnect(); + } catch { + /* ignore */ + } + } +} + +export default function makeClient(): ChatClient { + return new SpacetimeChatClient(); +} diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/index.ts b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/index.ts new file mode 100644 index 00000000000..89ad1908bc4 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/index.ts @@ -0,0 +1,122 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb cli version 2.2.0 (commit eb11e2f5c41dce6979715ad407996270d61329f6). + +/* eslint-disable */ +/* tslint:disable */ +import { + DbConnectionBuilder as __DbConnectionBuilder, + DbConnectionImpl as __DbConnectionImpl, + SubscriptionBuilderImpl as __SubscriptionBuilderImpl, + TypeBuilder as __TypeBuilder, + Uuid as __Uuid, + convertToAccessorMap as __convertToAccessorMap, + makeQueryBuilder as __makeQueryBuilder, + procedureSchema as __procedureSchema, + procedures as __procedures, + reducerSchema as __reducerSchema, + reducers as __reducers, + schema as __schema, + t as __t, + table as __table, + type AlgebraicTypeType as __AlgebraicTypeType, + type DbConnectionConfig as __DbConnectionConfig, + type ErrorContextInterface as __ErrorContextInterface, + type Event as __Event, + type EventContextInterface as __EventContextInterface, + type Infer as __Infer, + type QueryBuilder as __QueryBuilder, + type ReducerEventContextInterface as __ReducerEventContextInterface, + type RemoteModule as __RemoteModule, + type SubscriptionEventContextInterface as __SubscriptionEventContextInterface, + type SubscriptionHandleImpl as __SubscriptionHandleImpl, +} from "spacetimedb"; + +// Import all reducer arg schemas +import SendMessageReducer from "./send_message_reducer"; + +// Import all procedure arg schemas + +// Import all table schema definitions +import MessageRow from "./message_table"; + +/** Type-only namespace exports for generated type groups. */ + +/** The schema information for all tables in this module. This is defined the same was as the tables would have been defined in the server. */ +const tablesSchema = __schema({ + message: __table({ + name: 'message', + indexes: [ + { accessor: 'id', name: 'message_id_idx_btree', algorithm: 'btree', columns: [ + 'id', + ] }, + ], + constraints: [ + { name: 'message_id_key', constraint: 'unique', columns: ['id'] }, + ], + }, MessageRow), +}); + +/** The schema information for all reducers in this module. This is defined the same way as the reducers would have been defined in the server, except the body of the reducer is omitted in code generation. */ +const reducersSchema = __reducers( + __reducerSchema("send_message", SendMessageReducer), +); + +/** The schema information for all procedures in this module. This is defined the same way as the procedures would have been defined in the server. */ +const proceduresSchema = __procedures( +); + +/** The remote SpacetimeDB module schema, both runtime and type information. */ +const REMOTE_MODULE = { + versionInfo: { + cliVersion: "2.2.0" as const, + }, + tables: tablesSchema.schemaType.tables, + reducers: reducersSchema.reducersType.reducers, + ...proceduresSchema, +} satisfies __RemoteModule< + typeof tablesSchema.schemaType, + typeof reducersSchema.reducersType, + typeof proceduresSchema +>; + +/** The tables available in this remote SpacetimeDB module. Each table reference doubles as a query builder. */ +export const tables: __QueryBuilder = __makeQueryBuilder(tablesSchema.schemaType); + +/** The reducers available in this remote SpacetimeDB module. */ +export const reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers); + +/** The procedures available in this remote SpacetimeDB module. */ +export const procedures = __convertToAccessorMap(proceduresSchema.procedures); + +/** The context type returned in callbacks for all possible events. */ +export type EventContext = __EventContextInterface; +/** The context type returned in callbacks for reducer events. */ +export type ReducerEventContext = __ReducerEventContextInterface; +/** The context type returned in callbacks for subscription events. */ +export type SubscriptionEventContext = __SubscriptionEventContextInterface; +/** The context type returned in callbacks for error events. */ +export type ErrorContext = __ErrorContextInterface; +/** The subscription handle type to manage active subscriptions created from a {@link SubscriptionBuilder}. */ +export type SubscriptionHandle = __SubscriptionHandleImpl; + +/** Builder class to configure a new subscription to the remote SpacetimeDB instance. */ +export class SubscriptionBuilder extends __SubscriptionBuilderImpl {} + +/** Builder class to configure a new database connection to the remote SpacetimeDB instance. */ +export class DbConnectionBuilder extends __DbConnectionBuilder {} + +/** The typed database connection to manage connections to the remote SpacetimeDB instance. This class has type information specific to the generated module. */ +export class DbConnection extends __DbConnectionImpl { + /** Creates a new {@link DbConnectionBuilder} to configure and connect to the remote SpacetimeDB instance. */ + static builder = (): DbConnectionBuilder => { + return new DbConnectionBuilder(REMOTE_MODULE, (config: __DbConnectionConfig) => new DbConnection(config)); + }; + + /** Creates a new {@link SubscriptionBuilder} to configure a subscription to the remote SpacetimeDB instance. */ + override subscriptionBuilder = (): SubscriptionBuilder => { + return new SubscriptionBuilder(this); + }; +} + diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/message_table.ts b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/message_table.ts new file mode 100644 index 00000000000..b4779fb3e8c --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/message_table.ts @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64().primaryKey(), + sender: __t.string(), + text: __t.string(), + sentAt: __t.timestamp().name("sent_at"), +}); diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/send_message_reducer.ts b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/send_message_reducer.ts new file mode 100644 index 00000000000..d6fcf2a43f6 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/send_message_reducer.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + sender: __t.string(), + text: __t.string(), +}; diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/types.ts b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/types.ts new file mode 100644 index 00000000000..9d8856f074d --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/types.ts @@ -0,0 +1,20 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export const Message = __t.object("Message", { + id: __t.u64(), + sender: __t.string(), + text: __t.string(), + sentAt: __t.timestamp(), +}); +export type Message = __Infer; + diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/types/procedures.ts b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/types/procedures.ts new file mode 100644 index 00000000000..d5ac825c9ab --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/types/procedures.ts @@ -0,0 +1,10 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { type Infer as __Infer } from "spacetimedb"; + +// Import all procedure arg schemas + + diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/types/reducers.ts b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/types/reducers.ts new file mode 100644 index 00000000000..b6e7bae3b0d --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/module_bindings/types/reducers.ts @@ -0,0 +1,12 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { type Infer as __Infer } from "spacetimedb"; + +// Import all reducer arg schemas +import SendMessageReducer from "../send_message_reducer"; + +export type SendMessageParams = __Infer; + diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/package.json b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/package.json new file mode 100644 index 00000000000..f3fe4ec6978 --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/package.json @@ -0,0 +1,13 @@ +{ + "name": "stack-bench-verifier-spacetimedb", + "version": "0.0.1", + "private": true, + "type": "module", + "description": "SpacetimeDB verifier: runs the shared behavioral scenario via the SpacetimeDB TS SDK. Versions pinned to match the committed module_bindings (CLI/SDK 2.2.0).", + "dependencies": { + "spacetimedb": "2.2.0" + }, + "devDependencies": { + "tsx": "^4.19.0" + } +} diff --git a/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/test.sh b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/test.sh new file mode 100755 index 00000000000..b94e4aa4b3b --- /dev/null +++ b/tools/stack-bench/tasks/realtime-chat/spacetimedb/tests/test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Thin launcher — Harbor's verifier entry point (Harbor requires test.sh on Linux). +# Runs the shared behavioral scenario through the SpacetimeDB adapter; the harness +# writes the reward to /logs/verifier/reward.txt. Connects to localhost (SHARED mode). +set -euo pipefail +cd "$(dirname "$0")" +npm install --no-audit --no-fund --silent +exec npx tsx harness/src/runScenario.ts "$(pwd)/adapter.ts" diff --git a/tools/stack-bench/tasks/team-chat/convex/environment/Dockerfile b/tools/stack-bench/tasks/team-chat/convex/environment/Dockerfile new file mode 100644 index 00000000000..d87ec71fcc9 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/environment/Dockerfile @@ -0,0 +1,22 @@ +# Environment the agent works in: the official self-hosted Convex backend image +# (Ubuntu 24.04, /convex/convex-local-backend + run scripts) with Node 22 added +# for the agent's `npx convex` tooling and the verifier. Single container so +# the verifier can kill and restart the backend process for durability checks. +FROM ghcr.io/get-convex/convex-backend:latest + +USER root +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates procps \ + && curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +COPY backendctl /opt/stack-bench/backendctl +RUN chmod +x /opt/stack-bench/backendctl && mkdir -p /var/log + +WORKDIR /app +EXPOSE 3210 3211 + +# Boot the backend via backendctl, then exec Harbor's keepalive command so the +# agent and the SHARED-mode verifier share this container with the backend up. +ENTRYPOINT ["/bin/sh", "-c", "/opt/stack-bench/backendctl start && exec \"$@\"", "sh"] +CMD ["sh", "-c", "sleep infinity"] diff --git a/tools/stack-bench/tasks/team-chat/convex/environment/backendctl b/tools/stack-bench/tasks/team-chat/convex/environment/backendctl new file mode 100644 index 00000000000..a916f1ca8ed --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/environment/backendctl @@ -0,0 +1,55 @@ +#!/bin/sh +# Backend lifecycle control for the in-container self-hosted Convex backend. +# Used by the Dockerfile ENTRYPOINT at boot and by the verifier's durability +# phase (RESTART_CMD="/opt/stack-bench/backendctl restart"). SQLite state under +# /convex/data survives process restarts, which is what the durability checks +# verify. +set -eu + +LOG=/var/log/convex.log + +start() { + cd /convex + # Fixed name+secret -> deterministic admin key; localhost origins because the + # backend, agent, and verifier all share this one container. + INSTANCE_NAME="${INSTANCE_NAME:-convex-stack-bench}" \ + INSTANCE_SECRET="${INSTANCE_SECRET:-0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef}" \ + CONVEX_CLOUD_ORIGIN="${CONVEX_CLOUD_ORIGIN:-http://127.0.0.1:3210}" \ + CONVEX_SITE_ORIGIN="${CONVEX_SITE_ORIGIN:-http://127.0.0.1:3211}" \ + DISABLE_BEACON=1 \ + nohup ./run_backend.sh >>"$LOG" 2>&1 & +} + +stop() { + pkill -f convex-local-backend 2>/dev/null || true + i=0 + while curl -s -o /dev/null --max-time 1 http://127.0.0.1:3210/version 2>/dev/null; do + i=$((i + 1)) + [ "$i" -ge 30 ] && { echo "backendctl: backend did not stop" >&2; exit 1; } + sleep 0.5 + done +} + +wait_ready() { + i=0 + until curl -sf -o /dev/null --max-time 2 http://127.0.0.1:3210/version 2>/dev/null; do + i=$((i + 1)) + [ "$i" -ge 120 ] && { echo "backendctl: backend did not become ready" >&2; exit 1; } + sleep 1 + done +} + +case "${1:-}" in + start) start ;; + stop) stop ;; + wait-ready) wait_ready ;; + restart) + stop + start + wait_ready + ;; + *) + echo "usage: backendctl {start|stop|restart|wait-ready}" >&2 + exit 2 + ;; +esac diff --git a/tools/stack-bench/tasks/team-chat/convex/instruction.md b/tools/stack-bench/tasks/team-chat/convex/instruction.md new file mode 100644 index 00000000000..eb0b20234ad --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/instruction.md @@ -0,0 +1,190 @@ + + +# Task: team chat backend + +Build the backend for a team chat application (think a minimal Slack/Discord +server): named rooms with owners and members, real-time messaging with +editing and deletion, per-member unread counters, user presence, and a credit +system where users can tip each other. + +The application is graded **behaviorally** by an automated verifier that +drives the running backend through its real client SDK with multiple +concurrent clients. It checks functional correctness, transactional +correctness under concurrency, real-time delivery, durability across a +backend restart, and basic performance. Partial credit is awarded per check. +Grading does not inspect your source code — only observable behavior counts. + +## Data model (required behavior, not storage advice) + +- **User** — `username` (unique string), `status` (one of `"online"`, + `"away"`, `"offline"`), `balance` (integer credits). +- **Room** — `name` (unique string), `owner` (a username). +- **Membership** — which users are in which rooms, plus per-member + `last_read_seq` and `unread` counters (see Unread rules). +- **Message** — belongs to a room; has a server-assigned per-room sequence + number `seq`, a client-supplied id `client_msg_id`, `sender`, `text`, + `edited` flag, `deleted` flag. + +## Operations + +All operations validate their inputs and **fail with an error** when a rule +below is violated. "Fails" must be observable to the caller through the +backend's normal error mechanism (rejected mutation / failed reducer call). + +1. **register(username)** — Creates the user with `status = "online"` and + `balance = 100`. If the user already exists, the call **succeeds** and + leaves the existing user completely unchanged (idempotent; must NOT reset + balance or status). +2. **set_status(username, status)** — Sets presence. Fails for unknown users + or a status outside the three allowed values. +3. **create_room(username, room)** — Creates a room owned by `username`, who + automatically becomes a member (with `last_read_seq = 0`, `unread = 0`). + Fails for unknown users or if the room name already exists. +4. **join_room(username, room)** — Adds the user as a member with + `last_read_seq = 0` and `unread` equal to the number of messages already + in the room sent by other users. Fails for unknown user/room. If already + a member, succeeds without changing the existing membership state. +5. **leave_room(username, room)** — Removes the membership. Fails for + unknown user/room, if not a member, or if the user is the room's owner + (owners cannot leave). +6. **kick(actor, room, target)** — Removes `target`'s membership. Fails + unless `actor` is the room's owner; fails if `target` is the owner or not + a member. +7. **send_message(sender, room, text, client_msg_id)** — Appends a message. + Fails for: unknown user/room, sender not a member, empty `text`, `text` + longer than 4000 characters. **Idempotency:** if a message with the same + `client_msg_id` already exists in the room, the call **succeeds** without + creating a second message (safe retry). Otherwise the server assigns + `seq` (see Sequence rules), stores the message with `edited = false`, + `deleted = false`, and **atomically** increments `unread` by 1 for every + member of the room except the sender. +8. **edit_message(actor, room, client_msg_id, new_text)** — Replaces the + text and sets `edited = true`. Fails unless `actor` is the original + sender; fails if the message is deleted or `new_text` fails the same + validation as send. +9. **delete_message(actor, room, client_msg_id)** — Tombstones the message: + sets `deleted = true` and **clears `text` to the empty string**. Allowed + for the original sender or the room owner; fails for anyone else or if + already deleted. The original text must not be recoverable by any client + afterwards (late-joining clients must see the tombstone, never the text). +10. **mark_read(user, room, up_to_seq)** — Sets the member's + `last_read_seq = max(current, up_to_seq)` (it never decreases), then + recomputes `unread` per the Unread rules. Fails if not a member. +11. **tip(from_user, to_user, amount)** — Atomically transfers `amount` + credits. Fails for: unknown users, `from_user == to_user`, + `amount <= 0`, or `from_user`'s balance below `amount`. Balances must + never go negative and total credits must be conserved, including under + concurrent tips. + +## Sequence rules (transactional correctness) + +- `seq` is **per room**, assigned by the server, starting at 1 for the + room's first message and increasing by exactly 1 per stored message — + **no gaps, no duplicates**, even when many clients send concurrently. +- The counter must survive a backend restart: the first message stored + after a restart continues from the pre-restart maximum (durable counter — + an in-memory counter that resets and reuses sequence numbers fails). + +## Unread rules + +At any quiescent moment, for every member `u` of room `r`: +`unread(u, r) == count of messages m in r with m.seq > last_read_seq(u, r) and m.sender != u`. +Deleted (tombstoned) messages still count — they were sent. This invariant +must hold exactly, including after concurrent sends from multiple clients +(the per-message unread increments must be atomic with message insertion). + +## Real-time requirements + +Connected clients observe changes via **push** (the backend's real-time +subscription mechanism, not client polling): + +- A client subscribed to a room receives every existing message at + subscribe time (in `seq` order, reflecting current edited/deleted state) + and every subsequent message thereafter, exactly once, in `seq` order. +- Message **edits and deletions** are pushed to subscribers in real time. +- Membership changes (join / leave / kick / read-state updates) are pushed + to subscribers of the room's membership. +- User changes (status, balance) are pushed to subscribers of the user list. +- Subscriptions are **per room**: a client subscribed to room A must not + receive room B's messages. + +## Durability requirements + +All state — users, balances, rooms, memberships, read state, messages +(including edits and tombstones), the idempotency record of seen +`client_msg_id`s, and the per-room sequence counters — must survive a +backend process restart. The verifier will restart the backend mid-run, +reconnect fresh clients, and check every piece of state plus continued +real-time operation. + +## Performance requirements + +Modest but real: with a handful of clients, message delivery latency +(send-call to another subscribed client receiving the push) should be well +under 1.5 seconds at p95, and a burst of 120 messages from 3 concurrent +senders should be fully delivered to a subscriber within 45 seconds. A +correct implementation on this backend passes these comfortably; polling +loops with long intervals do not. + +## What is graded + +The verifier awards partial credit across four weighted groups: +correctness & transactions (0.40), real-time behavior (0.30), durability +across restart (0.20), performance (0.10). Every check is machine-verified +through the client SDK against the running backend. + +Your implementation MUST expose exactly the identifiers named in the +**Contract** section below so the grader can connect. + +## Contract (Convex) + +A self-hosted Convex backend is already running in this container at +`http://127.0.0.1:3210`. Build a Convex app and deploy it with: + + export CONVEX_SELF_HOSTED_URL=http://127.0.0.1:3210 + export CONVEX_SELF_HOSTED_ADMIN_KEY='convex-stack-bench|01be067fd1488e360c17a915fd342953e6450766d4831138261df4371e27342009f370391e' + npx convex deploy -y + +The grader connects with the official Convex client SDK and calls your +functions **by these exact names** (module.function) with these argument +object keys. You may design your table schema freely, but every function +below must exist with these signatures and behaviors: + +Mutations (throw an `Error` to report failures): + + users.register({ username }) + users.setStatus({ username, status }) + rooms.createRoom({ username, room }) + rooms.joinRoom({ username, room }) + rooms.leaveRoom({ username, room }) + rooms.kick({ actor, room, target }) + messages.send({ sender, room, text, clientMsgId }) + messages.edit({ actor, room, clientMsgId, newText }) + messages.remove({ actor, room, clientMsgId }) + messages.markRead({ user, room, upToSeq }) + credits.tip({ fromUser, toUser, amount }) + +Queries (all reactive — the grader subscribes to them via the SDK's onUpdate +and expects pushed updates on every relevant change): + + users.get({ username }) -> { username, status, balance } | null + users.list({}) -> [{ username, status, balance }] + rooms.getOwner({ room }) -> string | null + rooms.listMembers({ room }) -> [{ user, lastReadSeq, unread }] sorted by user + messages.list({ room }) -> [{ seq, clientMsgId, sender, text, edited, deleted }] + sorted by seq ascending + +Notes: + +- Convex mutations are serializable transactions; use that to keep the + per-room `seq` counter gapless and the unread increments atomic with the + message insert, including under concurrent calls. +- The backend stores state in SQLite under /convex/data; the verifier + restarts the backend process and expects all state (including per-room seq + counters and clientMsgId dedupe behavior) to survive. +- Do not modify or stop the running backend; deploy your app to it. diff --git a/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/credits.ts b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/credits.ts new file mode 100644 index 00000000000..ae830aa135a --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/credits.ts @@ -0,0 +1,19 @@ +// Reference (oracle) credit-transfer function. A single Convex mutation is a +// serializable transaction, so the two balance writes are atomic and total +// credits are conserved under concurrency. +import { mutation } from "./_generated/server"; +import { v } from "convex/values"; +import { getUserDoc } from "./lib"; + +export const tip = mutation({ + args: { fromUser: v.string(), toUser: v.string(), amount: v.number() }, + handler: async (ctx, { fromUser, toUser, amount }) => { + if (fromUser === toUser) throw new Error("cannot tip yourself"); + if (!Number.isInteger(amount) || amount <= 0) throw new Error("amount must be positive"); + const from = await getUserDoc(ctx, fromUser); + const to = await getUserDoc(ctx, toUser); + if (from.balance < amount) throw new Error("insufficient balance"); + await ctx.db.patch(from._id, { balance: from.balance - amount }); + await ctx.db.patch(to._id, { balance: to.balance + amount }); + }, +}); diff --git a/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/lib.ts b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/lib.ts new file mode 100644 index 00000000000..950b45963e5 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/lib.ts @@ -0,0 +1,35 @@ +// Shared helpers for the oracle's Convex functions. +import type { MutationCtx, QueryCtx } from "./_generated/server"; + +export const STATUSES = ["online", "away", "offline"]; +export const MAX_TEXT = 4000; + +export async function getUserDoc(ctx: QueryCtx | MutationCtx, username: string) { + const doc = await ctx.db + .query("users") + .withIndex("by_username", (q) => q.eq("username", username)) + .unique(); + if (!doc) throw new Error(`unknown user: ${username}`); + return doc; +} + +export async function getRoomDoc(ctx: QueryCtx | MutationCtx, name: string) { + const doc = await ctx.db + .query("rooms") + .withIndex("by_name", (q) => q.eq("name", name)) + .unique(); + if (!doc) throw new Error(`unknown room: ${name}`); + return doc; +} + +export async function findMember(ctx: QueryCtx | MutationCtx, room: string, user: string) { + return await ctx.db + .query("members") + .withIndex("by_room_user", (q) => q.eq("room", room).eq("user", user)) + .unique(); +} + +export function validateText(text: string) { + if (text.length === 0) throw new Error("text must not be empty"); + if (text.length > MAX_TEXT) throw new Error(`text exceeds ${MAX_TEXT} characters`); +} diff --git a/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/messages.ts b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/messages.ts new file mode 100644 index 00000000000..97f4416ad1e --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/messages.ts @@ -0,0 +1,115 @@ +// Reference (oracle) message functions: idempotent sends, gapless per-room +// seq, atomic unread increments, edit, tombstone delete, monotone mark-read. +// Convex mutations are serializable transactions (OCC with retries), so the +// read-increment-write on rooms.nextSeq and the multi-row unread updates are +// atomic and the concurrency checks hold. +import { mutation, query } from "./_generated/server"; +import { v } from "convex/values"; +import { findMember, getRoomDoc, getUserDoc, validateText } from "./lib"; + +export const send = mutation({ + args: { sender: v.string(), room: v.string(), text: v.string(), clientMsgId: v.string() }, + handler: async (ctx, { sender, room, text, clientMsgId }) => { + await getUserDoc(ctx, sender); + const r = await getRoomDoc(ctx, room); + if (!(await findMember(ctx, room, sender))) throw new Error("sender is not a member"); + validateText(text); + // Idempotent retry: same clientMsgId in this room -> success, no new row. + const dupe = await ctx.db + .query("messages") + .withIndex("by_room_client", (q) => q.eq("room", room).eq("clientMsgId", clientMsgId)) + .unique(); + if (dupe) return; + const seq = r.nextSeq; + await ctx.db.patch(r._id, { nextSeq: seq + 1 }); + await ctx.db.insert("messages", { + room, + seq, + clientMsgId, + sender, + text, + edited: false, + deleted: false, + sentAt: Date.now(), + }); + const members = await ctx.db + .query("members") + .withIndex("by_room", (q) => q.eq("room", room)) + .collect(); + for (const m of members) { + if (m.user !== sender) await ctx.db.patch(m._id, { unread: m.unread + 1 }); + } + }, +}); + +async function findMessage(ctx: any, room: string, clientMsgId: string) { + const msg = await ctx.db + .query("messages") + .withIndex("by_room_client", (q: any) => q.eq("room", room).eq("clientMsgId", clientMsgId)) + .unique(); + if (!msg) throw new Error(`no message ${clientMsgId} in ${room}`); + return msg; +} + +export const edit = mutation({ + args: { actor: v.string(), room: v.string(), clientMsgId: v.string(), newText: v.string() }, + handler: async (ctx, { actor, room, clientMsgId, newText }) => { + await getUserDoc(ctx, actor); + await getRoomDoc(ctx, room); + const msg = await findMessage(ctx, room, clientMsgId); + if (msg.sender !== actor) throw new Error("only the original sender can edit"); + if (msg.deleted) throw new Error("cannot edit a deleted message"); + validateText(newText); + await ctx.db.patch(msg._id, { text: newText, edited: true }); + }, +}); + +export const remove = mutation({ + args: { actor: v.string(), room: v.string(), clientMsgId: v.string() }, + handler: async (ctx, { actor, room, clientMsgId }) => { + await getUserDoc(ctx, actor); + const r = await getRoomDoc(ctx, room); + const msg = await findMessage(ctx, room, clientMsgId); + if (msg.sender !== actor && r.owner !== actor) { + throw new Error("only the sender or the room owner can delete"); + } + if (msg.deleted) throw new Error("message is already deleted"); + await ctx.db.patch(msg._id, { deleted: true, text: "" }); + }, +}); + +export const markRead = mutation({ + args: { user: v.string(), room: v.string(), upToSeq: v.number() }, + handler: async (ctx, { user, room, upToSeq }) => { + await getUserDoc(ctx, user); + await getRoomDoc(ctx, room); + const member = await findMember(ctx, room, user); + if (!member) throw new Error("not a member"); + // Monotone: lastReadSeq never decreases. + const lastReadSeq = Math.max(member.lastReadSeq, upToSeq); + const msgs = await ctx.db + .query("messages") + .withIndex("by_room_seq", (q) => q.eq("room", room)) + .collect(); + const unread = msgs.filter((m) => m.seq > lastReadSeq && m.sender !== user).length; + await ctx.db.patch(member._id, { lastReadSeq, unread }); + }, +}); + +export const list = query({ + args: { room: v.string() }, + handler: async (ctx, { room }) => { + const docs = await ctx.db + .query("messages") + .withIndex("by_room_seq", (q) => q.eq("room", room)) + .collect(); + return docs.map((d) => ({ + seq: d.seq, + clientMsgId: d.clientMsgId, + sender: d.sender, + text: d.text, + edited: d.edited, + deleted: d.deleted, + })); + }, +}); diff --git a/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/rooms.ts b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/rooms.ts new file mode 100644 index 00000000000..bc93c5d0fed --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/rooms.ts @@ -0,0 +1,84 @@ +// Reference (oracle) room/membership functions. +import { mutation, query } from "./_generated/server"; +import { v } from "convex/values"; +import { findMember, getRoomDoc, getUserDoc } from "./lib"; + +export const createRoom = mutation({ + args: { username: v.string(), room: v.string() }, + handler: async (ctx, { username, room }) => { + await getUserDoc(ctx, username); + if (room.length === 0) throw new Error("room name must not be empty"); + const existing = await ctx.db + .query("rooms") + .withIndex("by_name", (q) => q.eq("name", room)) + .unique(); + if (existing) throw new Error(`room already exists: ${room}`); + await ctx.db.insert("rooms", { name: room, owner: username, nextSeq: 1 }); + await ctx.db.insert("members", { room, user: username, lastReadSeq: 0, unread: 0 }); + }, +}); + +export const joinRoom = mutation({ + args: { username: v.string(), room: v.string() }, + handler: async (ctx, { username, room }) => { + await getUserDoc(ctx, username); + await getRoomDoc(ctx, room); + // Idempotent: existing membership state is preserved. + if (await findMember(ctx, room, username)) return; + const msgs = await ctx.db + .query("messages") + .withIndex("by_room_seq", (q) => q.eq("room", room)) + .collect(); + const unread = msgs.filter((m) => m.sender !== username).length; + await ctx.db.insert("members", { room, user: username, lastReadSeq: 0, unread }); + }, +}); + +export const leaveRoom = mutation({ + args: { username: v.string(), room: v.string() }, + handler: async (ctx, { username, room }) => { + await getUserDoc(ctx, username); + const r = await getRoomDoc(ctx, room); + if (r.owner === username) throw new Error("the room owner cannot leave"); + const member = await findMember(ctx, room, username); + if (!member) throw new Error("not a member"); + await ctx.db.delete(member._id); + }, +}); + +export const kick = mutation({ + args: { actor: v.string(), room: v.string(), target: v.string() }, + handler: async (ctx, { actor, room, target }) => { + await getUserDoc(ctx, actor); + const r = await getRoomDoc(ctx, room); + if (r.owner !== actor) throw new Error("only the room owner can kick"); + if (target === r.owner) throw new Error("cannot kick the room owner"); + const member = await findMember(ctx, room, target); + if (!member) throw new Error("target is not a member"); + await ctx.db.delete(member._id); + }, +}); + +export const getOwner = query({ + args: { room: v.string() }, + handler: async (ctx, { room }) => { + const doc = await ctx.db + .query("rooms") + .withIndex("by_name", (q) => q.eq("name", room)) + .unique(); + return doc ? doc.owner : null; + }, +}); + +export const listMembers = query({ + args: { room: v.string() }, + handler: async (ctx, { room }) => { + const docs = await ctx.db + .query("members") + .withIndex("by_room", (q) => q.eq("room", room)) + .collect(); + return docs + .map((d) => ({ user: d.user, lastReadSeq: d.lastReadSeq, unread: d.unread })) + .sort((a, b) => (a.user < b.user ? -1 : a.user > b.user ? 1 : 0)); + }, +}); diff --git a/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/schema.ts b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/schema.ts new file mode 100644 index 00000000000..0b95872377d --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/schema.ts @@ -0,0 +1,39 @@ +// Reference (oracle) Convex schema for the stack-bench team-chat task. +import { defineSchema, defineTable } from "convex/server"; +import { v } from "convex/values"; + +export default defineSchema({ + users: defineTable({ + username: v.string(), + status: v.string(), + balance: v.number(), + }).index("by_username", ["username"]), + + rooms: defineTable({ + name: v.string(), + owner: v.string(), + nextSeq: v.number(), + }).index("by_name", ["name"]), + + members: defineTable({ + room: v.string(), + user: v.string(), + lastReadSeq: v.number(), + unread: v.number(), + }) + .index("by_room", ["room"]) + .index("by_room_user", ["room", "user"]), + + messages: defineTable({ + room: v.string(), + seq: v.number(), + clientMsgId: v.string(), + sender: v.string(), + text: v.string(), + edited: v.boolean(), + deleted: v.boolean(), + sentAt: v.number(), + }) + .index("by_room_seq", ["room", "seq"]) + .index("by_room_client", ["room", "clientMsgId"]), +}); diff --git a/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/users.ts b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/users.ts new file mode 100644 index 00000000000..cb1885c933e --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/solution/app/convex/users.ts @@ -0,0 +1,47 @@ +// Reference (oracle) user/presence functions. +import { mutation, query } from "./_generated/server"; +import { v } from "convex/values"; +import { getUserDoc, STATUSES } from "./lib"; + +export const register = mutation({ + args: { username: v.string() }, + handler: async (ctx, { username }) => { + if (username.length === 0) throw new Error("username must not be empty"); + const existing = await ctx.db + .query("users") + .withIndex("by_username", (q) => q.eq("username", username)) + .unique(); + // Idempotent: existing users are left completely unchanged. + if (!existing) { + await ctx.db.insert("users", { username, status: "online", balance: 100 }); + } + }, +}); + +export const setStatus = mutation({ + args: { username: v.string(), status: v.string() }, + handler: async (ctx, { username, status }) => { + if (!STATUSES.includes(status)) throw new Error(`invalid status: ${status}`); + const user = await getUserDoc(ctx, username); + await ctx.db.patch(user._id, { status }); + }, +}); + +export const get = query({ + args: { username: v.string() }, + handler: async (ctx, { username }) => { + const doc = await ctx.db + .query("users") + .withIndex("by_username", (q) => q.eq("username", username)) + .unique(); + return doc ? { username: doc.username, status: doc.status, balance: doc.balance } : null; + }, +}); + +export const list = query({ + args: {}, + handler: async (ctx) => { + const docs = await ctx.db.query("users").collect(); + return docs.map((d) => ({ username: d.username, status: d.status, balance: d.balance })); + }, +}); diff --git a/tools/stack-bench/tasks/team-chat/convex/solution/app/package-lock.json b/tools/stack-bench/tasks/team-chat/convex/solution/app/package-lock.json new file mode 100644 index 00000000000..bbf3c8d0e4d --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/solution/app/package-lock.json @@ -0,0 +1,546 @@ +{ + "name": "stack-bench-teamchat-convex-oracle", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "stack-bench-teamchat-convex-oracle", + "version": "0.0.1", + "dependencies": { + "convex": "^1.17.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.0.tgz", + "integrity": "sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.0.tgz", + "integrity": "sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.0.tgz", + "integrity": "sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.0.tgz", + "integrity": "sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.0.tgz", + "integrity": "sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.0.tgz", + "integrity": "sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.0.tgz", + "integrity": "sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.0.tgz", + "integrity": "sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.0.tgz", + "integrity": "sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.0.tgz", + "integrity": "sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.0.tgz", + "integrity": "sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.0.tgz", + "integrity": "sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.0.tgz", + "integrity": "sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.0.tgz", + "integrity": "sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.0.tgz", + "integrity": "sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.0.tgz", + "integrity": "sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.0.tgz", + "integrity": "sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.0.tgz", + "integrity": "sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.0.tgz", + "integrity": "sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.0.tgz", + "integrity": "sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.0.tgz", + "integrity": "sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.0.tgz", + "integrity": "sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.0.tgz", + "integrity": "sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.0.tgz", + "integrity": "sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.0.tgz", + "integrity": "sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.0.tgz", + "integrity": "sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/convex": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/convex/-/convex-1.43.0.tgz", + "integrity": "sha512-huZWEUZQYxIRG104o22ZI99HkviSEVltwezZRDi+pQDPrQbc5EoCPa4Y7pJDqUBQw9dc/iEEXj2/rLZg1j7Dww==", + "license": "Apache-2.0", + "dependencies": { + "esbuild": "0.27.0", + "prettier": "^3.0.0", + "ws": "8.21.0" + }, + "bin": { + "convex": "bin/main.js" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=7.0.0" + }, + "peerDependencies": { + "@auth0/auth0-react": "^2.0.1", + "@clerk/clerk-react": "^4.12.8 || ^5.0.0", + "@clerk/react": "^6.4.3", + "react": "^18.0.0 || ^19.0.0-0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@auth0/auth0-react": { + "optional": true + }, + "@clerk/clerk-react": { + "optional": true + }, + "@clerk/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/esbuild": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.0.tgz", + "integrity": "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.0", + "@esbuild/android-arm": "0.27.0", + "@esbuild/android-arm64": "0.27.0", + "@esbuild/android-x64": "0.27.0", + "@esbuild/darwin-arm64": "0.27.0", + "@esbuild/darwin-x64": "0.27.0", + "@esbuild/freebsd-arm64": "0.27.0", + "@esbuild/freebsd-x64": "0.27.0", + "@esbuild/linux-arm": "0.27.0", + "@esbuild/linux-arm64": "0.27.0", + "@esbuild/linux-ia32": "0.27.0", + "@esbuild/linux-loong64": "0.27.0", + "@esbuild/linux-mips64el": "0.27.0", + "@esbuild/linux-ppc64": "0.27.0", + "@esbuild/linux-riscv64": "0.27.0", + "@esbuild/linux-s390x": "0.27.0", + "@esbuild/linux-x64": "0.27.0", + "@esbuild/netbsd-arm64": "0.27.0", + "@esbuild/netbsd-x64": "0.27.0", + "@esbuild/openbsd-arm64": "0.27.0", + "@esbuild/openbsd-x64": "0.27.0", + "@esbuild/openharmony-arm64": "0.27.0", + "@esbuild/sunos-x64": "0.27.0", + "@esbuild/win32-arm64": "0.27.0", + "@esbuild/win32-ia32": "0.27.0", + "@esbuild/win32-x64": "0.27.0" + } + }, + "node_modules/prettier": { + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.6.tgz", + "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/tools/stack-bench/tasks/team-chat/convex/solution/app/package.json b/tools/stack-bench/tasks/team-chat/convex/solution/app/package.json new file mode 100644 index 00000000000..cc032168839 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/solution/app/package.json @@ -0,0 +1,9 @@ +{ + "name": "stack-bench-teamchat-convex-oracle", + "version": "0.0.1", + "private": true, + "description": "Reference (oracle) Convex app for the stack-bench team-chat task.", + "dependencies": { + "convex": "^1.17.0" + } +} diff --git a/tools/stack-bench/tasks/team-chat/convex/solution/solve.sh b/tools/stack-bench/tasks/team-chat/convex/solution/solve.sh new file mode 100644 index 00000000000..ab53e882653 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/solution/solve.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Thin launcher — Harbor's oracle entry point. Deploys the reference Convex app +# to the in-container self-hosted backend. Readiness is gated by the task.toml +# healthcheck. +set -euo pipefail +cd "$(dirname "$0")/app" +npm install --no-audit --no-fund --silent +export CONVEX_SELF_HOSTED_URL="${CONVEX_SELF_HOSTED_URL:-http://127.0.0.1:3210}" +# Deterministic admin key for the fixed INSTANCE_NAME/SECRET in environment/backendctl. +export CONVEX_SELF_HOSTED_ADMIN_KEY="${CONVEX_SELF_HOSTED_ADMIN_KEY:-convex-stack-bench|01be067fd1488e360c17a915fd342953e6450766d4831138261df4371e27342009f370391e}" +exec npx convex deploy -y diff --git a/tools/stack-bench/tasks/team-chat/convex/task.toml b/tools/stack-bench/tasks/team-chat/convex/task.toml new file mode 100644 index 00000000000..12698cdd074 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/task.toml @@ -0,0 +1,42 @@ +schema_version = "1.2" +artifacts = [] + +[task] +name = "clockwork/team-chat-convex" +description = "Build a full team-chat backend (rooms, presence, unread counters, edits/tombstones, idempotent sends, gapless per-room seq, atomic tips) on Convex; graded behaviorally incl. concurrency, restart durability, and latency." +keywords = ["realtime", "chat", "convex", "cross-backend", "stack-bench", "durability", "acid"] +[[task.authors]] +name = "Clockwork Labs" +email = "" + +[metadata] +# stack-bench-specific: which backend this task variant exercises. +backend = "convex" +tags = ["convex"] +category = "team-chat" + +[verifier] +# The scenario runs ~44 checks including a backend restart and perf probes. +timeout_sec = 1200.0 +# SHARED mode (default with no [verifier.environment]): the verifier runs +# inside the agent's container and reaches the backend + backendctl directly. + +[agent] +timeout_sec = 3600.0 + +[environment] +build_timeout_sec = 1800.0 +os = "linux" +cpus = 2 +memory_mb = 4096 +storage_mb = 12288 +gpus = 0 +allow_internet = true + +# Gate the agent on the in-container Convex backend being ready. +[environment.healthcheck] +command = "curl -sf -o /dev/null http://127.0.0.1:3210/version" +interval_sec = 3.0 +timeout_sec = 10.0 +start_period_sec = 5.0 +retries = 40 diff --git a/tools/stack-bench/tasks/team-chat/convex/tests/adapter.ts b/tools/stack-bench/tasks/team-chat/convex/tests/adapter.ts new file mode 100644 index 00000000000..fa1ce06abe7 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/tests/adapter.ts @@ -0,0 +1,151 @@ +// Convex adapter for the stack-bench team-chat task. +// +// Implements the harness AppClient contract against the running self-hosted +// Convex backend via the official client SDK. Mutations reject with the +// mutation's thrown error (the contract's accept/reject semantics). +// +// Real-time: Convex pushes full reactive-query results on every change; the +// adapter diffs consecutive results to synthesize per-row events. The first +// delivery is the room's history in seq order (Convex query results are +// index-ordered), satisfying the history-then-live contract. + +import { ConvexClient } from "convex/browser"; +import { anyApi } from "convex/server"; +import type { + AppClient, + MemberEventHandlers, + MemberRecord, + MessageRecord, + RoomEventHandlers, + UserEventHandlers, + UserRecord, +} from "./harness/src/appClient.js"; + +const CONVEX_URL = process.env.CONVEX_URL ?? "http://127.0.0.1:3210"; + +class ConvexAppClient implements AppClient { + private client!: ConvexClient; + private unsubs: Array<() => void> = []; + + async connect(): Promise { + this.client = new ConvexClient(CONVEX_URL); + // Fail fast if the backend is unreachable (ConvexClient connects lazily). + await this.client.query(anyApi.users.get, { username: "__probe__" }); + } + + async close(): Promise { + for (const u of this.unsubs.splice(0)) { + try { + u(); + } catch { + /* ignore */ + } + } + await this.client?.close(); + } + + // ---- mutations ---- + register(username: string) { + return this.client.mutation(anyApi.users.register, { username }) as Promise; + } + setStatus(username: string, status: string) { + return this.client.mutation(anyApi.users.setStatus, { username, status }) as Promise; + } + createRoom(username: string, room: string) { + return this.client.mutation(anyApi.rooms.createRoom, { username, room }) as Promise; + } + joinRoom(username: string, room: string) { + return this.client.mutation(anyApi.rooms.joinRoom, { username, room }) as Promise; + } + leaveRoom(username: string, room: string) { + return this.client.mutation(anyApi.rooms.leaveRoom, { username, room }) as Promise; + } + kick(actor: string, room: string, target: string) { + return this.client.mutation(anyApi.rooms.kick, { actor, room, target }) as Promise; + } + sendMessage(sender: string, room: string, text: string, clientMsgId: string) { + return this.client.mutation(anyApi.messages.send, { sender, room, text, clientMsgId }) as Promise; + } + editMessage(actor: string, room: string, clientMsgId: string, newText: string) { + return this.client.mutation(anyApi.messages.edit, { actor, room, clientMsgId, newText }) as Promise; + } + deleteMessage(actor: string, room: string, clientMsgId: string) { + return this.client.mutation(anyApi.messages.remove, { actor, room, clientMsgId }) as Promise; + } + markRead(user: string, room: string, upToSeq: number) { + return this.client.mutation(anyApi.messages.markRead, { user, room, upToSeq }) as Promise; + } + tip(fromUser: string, toUser: string, amount: number) { + return this.client.mutation(anyApi.credits.tip, { fromUser, toUser, amount }) as Promise; + } + + // ---- subscriptions (diff full reactive results into per-row events) ---- + async subscribeRoom(room: string, handlers: RoomEventHandlers): Promise { + const seen = new Map(); // clientMsgId -> serialized record + const unsub = this.client.onUpdate(anyApi.messages.list, { room }, (rows: MessageRecord[]) => { + const ordered = [...rows].sort((a, b) => a.seq - b.seq); + for (const row of ordered) { + const key = JSON.stringify(row); + if (seen.get(row.clientMsgId) !== key) { + seen.set(row.clientMsgId, key); + handlers.onMessage(row); + } + } + }); + this.unsubs.push(unsub); + } + + async subscribeUsers(handlers: UserEventHandlers): Promise { + const seen = new Map(); + const unsub = this.client.onUpdate(anyApi.users.list, {}, (rows: UserRecord[]) => { + for (const row of rows) { + const key = JSON.stringify(row); + if (seen.get(row.username) !== key) { + seen.set(row.username, key); + handlers.onUser(row); + } + } + }); + this.unsubs.push(unsub); + } + + async subscribeMembers(room: string, handlers: MemberEventHandlers): Promise { + const seen = new Map(); + const unsub = this.client.onUpdate(anyApi.rooms.listMembers, { room }, (rows: MemberRecord[]) => { + const present = new Set(rows.map((r) => r.user)); + for (const user of [...seen.keys()]) { + if (!present.has(user)) { + seen.delete(user); + handlers.onMemberRemoved?.(user); + } + } + for (const row of rows) { + const key = JSON.stringify(row); + if (seen.get(row.user) !== key) { + seen.set(row.user, key); + handlers.onMember(row); + } + } + }); + this.unsubs.push(unsub); + } + + // ---- snapshots ---- + getUser(username: string): Promise { + return this.client.query(anyApi.users.get, { username }) as Promise; + } + getRoomOwner(room: string): Promise { + return this.client.query(anyApi.rooms.getOwner, { room }) as Promise; + } + getMembers(room: string): Promise { + return this.client.query(anyApi.rooms.listMembers, { room }) as Promise; + } + async getMessages(room: string): Promise { + const rows = (await this.client.query(anyApi.messages.list, { room })) as MessageRecord[]; + return rows.sort((a, b) => a.seq - b.seq); + } +} + +export default function makeClient(): AppClient { + return new ConvexAppClient(); +} diff --git a/tools/stack-bench/tasks/team-chat/convex/tests/package-lock.json b/tools/stack-bench/tasks/team-chat/convex/tests/package-lock.json new file mode 100644 index 00000000000..0965ba1bc45 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/tests/package-lock.json @@ -0,0 +1,1067 @@ +{ + "name": "stack-bench-verifier-teamchat-convex", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "stack-bench-verifier-teamchat-convex", + "version": "0.0.1", + "dependencies": { + "convex": "^1.17.0" + }, + "devDependencies": { + "tsx": "^4.19.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.27.0.tgz", + "integrity": "sha512-KuZrd2hRjz01y5JK9mEBSD3Vj3mbCvemhT466rSuJYeE/hjuBrHfjjcjMdTm/sz7au+++sdbJZJmuBwQLuw68A==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.27.0.tgz", + "integrity": "sha512-j67aezrPNYWJEOHUNLPj9maeJte7uSMM6gMoxfPC9hOg8N02JuQi/T7ewumf4tNvJadFkvLZMlAq73b9uwdMyQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.27.0.tgz", + "integrity": "sha512-CC3vt4+1xZrs97/PKDkl0yN7w8edvU2vZvAFGD16n9F0Cvniy5qvzRXjfO1l94efczkkQE6g1x0i73Qf5uthOQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.27.0.tgz", + "integrity": "sha512-wurMkF1nmQajBO1+0CJmcN17U4BP6GqNSROP8t0X/Jiw2ltYGLHpEksp9MpoBqkrFR3kv2/te6Sha26k3+yZ9Q==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.27.0.tgz", + "integrity": "sha512-uJOQKYCcHhg07DL7i8MzjvS2LaP7W7Pn/7uA0B5S1EnqAirJtbyw4yC5jQ5qcFjHK9l6o/MX9QisBg12kNkdHg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.27.0.tgz", + "integrity": "sha512-8mG6arH3yB/4ZXiEnXof5MK72dE6zM9cDvUcPtxhUZsDjESl9JipZYW60C3JGreKCEP+p8P/72r69m4AZGJd5g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.27.0.tgz", + "integrity": "sha512-9FHtyO988CwNMMOE3YIeci+UV+x5Zy8fI2qHNpsEtSF83YPBmE8UWmfYAQg6Ux7Gsmd4FejZqnEUZCMGaNQHQw==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.27.0.tgz", + "integrity": "sha512-zCMeMXI4HS/tXvJz8vWGexpZj2YVtRAihHLk1imZj4efx1BQzN76YFeKqlDr3bUWI26wHwLWPd3rwh6pe4EV7g==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.27.0.tgz", + "integrity": "sha512-t76XLQDpxgmq2cNXKTVEB7O7YMb42atj2Re2Haf45HkaUpjM2J0UuJZDuaGbPbamzZ7bawyGFUkodL+zcE+jvQ==", + "cpu": [ + "arm" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.27.0.tgz", + "integrity": "sha512-AS18v0V+vZiLJyi/4LphvBE+OIX682Pu7ZYNsdUHyUKSoRwdnOsMf6FDekwoAFKej14WAkOef3zAORJgAtXnlQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.27.0.tgz", + "integrity": "sha512-Mz1jxqm/kfgKkc/KLHC5qIujMvnnarD9ra1cEcrs7qshTUSksPihGrWHVG5+osAIQ68577Zpww7SGapmzSt4Nw==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.27.0.tgz", + "integrity": "sha512-QbEREjdJeIreIAbdG2hLU1yXm1uu+LTdzoq1KCo4G4pFOLlvIspBm36QrQOar9LFduavoWX2msNFAAAY9j4BDg==", + "cpu": [ + "loong64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.27.0.tgz", + "integrity": "sha512-sJz3zRNe4tO2wxvDpH/HYJilb6+2YJxo/ZNbVdtFiKDufzWq4JmKAiHy9iGoLjAV7r/W32VgaHGkk35cUXlNOg==", + "cpu": [ + "mips64el" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.27.0.tgz", + "integrity": "sha512-z9N10FBD0DCS2dmSABDBb5TLAyF1/ydVb+N4pi88T45efQ/w4ohr/F/QYCkxDPnkhkp6AIpIcQKQ8F0ANoA2JA==", + "cpu": [ + "ppc64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.27.0.tgz", + "integrity": "sha512-pQdyAIZ0BWIC5GyvVFn5awDiO14TkT/19FTmFcPdDec94KJ1uZcmFs21Fo8auMXzD4Tt+diXu1LW1gHus9fhFQ==", + "cpu": [ + "riscv64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.27.0.tgz", + "integrity": "sha512-hPlRWR4eIDDEci953RI1BLZitgi5uqcsjKMxwYfmi4LcwyWo2IcRP+lThVnKjNtk90pLS8nKdroXYOqW+QQH+w==", + "cpu": [ + "s390x" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.27.0.tgz", + "integrity": "sha512-1hBWx4OUJE2cab++aVZ7pObD6s+DK4mPGpemtnAORBvb5l/g5xFGk0vc0PjSkrDs0XaXj9yyob3d14XqvnQ4gw==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.27.0.tgz", + "integrity": "sha512-6m0sfQfxfQfy1qRuecMkJlf1cIzTOgyaeXaiVaaki8/v+WB+U4hc6ik15ZW6TAllRlg/WuQXxWj1jx6C+dfy3w==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.27.0.tgz", + "integrity": "sha512-xbbOdfn06FtcJ9d0ShxxvSn2iUsGd/lgPIO2V3VZIPDbEaIj1/3nBBe1AwuEZKXVXkMmpr6LUAgMkLD/4D2PPA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.27.0.tgz", + "integrity": "sha512-fWgqR8uNbCQ/GGv0yhzttj6sU/9Z5/Sv/VGU3F5OuXK6J6SlriONKrQ7tNlwBrJZXRYk5jUhuWvF7GYzGguBZQ==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.27.0.tgz", + "integrity": "sha512-aCwlRdSNMNxkGGqQajMUza6uXzR/U0dIl1QmLjPtRbLOx3Gy3otfFu/VjATy4yQzo9yFDGTxYDo1FfAD9oRD2A==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.27.0.tgz", + "integrity": "sha512-nyvsBccxNAsNYz2jVFYwEGuRRomqZ149A39SHWk4hV0jWxKM0hjBPm3AmdxcbHiFLbBSwG6SbpIcUbXjgyECfA==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.27.0.tgz", + "integrity": "sha512-Q1KY1iJafM+UX6CFEL+F4HRTgygmEW568YMqDA5UV97AuZSm21b7SXIrRJDwXWPzr8MGr75fUZPV67FdtMHlHA==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.27.0.tgz", + "integrity": "sha512-W1eyGNi6d+8kOmZIwi/EDjrL9nxQIQ0MiGqe/AWc6+IaHloxHSGoeRgDRKHFISThLmsewZ5nHFvGFWdBYlgKPg==", + "cpu": [ + "arm64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.27.0.tgz", + "integrity": "sha512-30z1aKL9h22kQhilnYkORFYt+3wp7yZsHWus+wSKAJR8JtdfI76LJ4SBdMsCopTR3z/ORqVu5L1vtnHZWVj4cQ==", + "cpu": [ + "ia32" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.27.0.tgz", + "integrity": "sha512-aIitBcjQeyOhMTImhLZmtxfdOcuNRpwlPNmlFKPcHQYPhEssw75Cl1TSXJXpMkzaua9FUetx/4OQKq7eJul5Cg==", + "cpu": [ + "x64" + ], + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/convex": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/convex/-/convex-1.43.0.tgz", + "integrity": "sha512-huZWEUZQYxIRG104o22ZI99HkviSEVltwezZRDi+pQDPrQbc5EoCPa4Y7pJDqUBQw9dc/iEEXj2/rLZg1j7Dww==", + "license": "Apache-2.0", + "dependencies": { + "esbuild": "0.27.0", + "prettier": "^3.0.0", + "ws": "8.21.0" + }, + "bin": { + "convex": "bin/main.js" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=7.0.0" + }, + "peerDependencies": { + "@auth0/auth0-react": "^2.0.1", + "@clerk/clerk-react": "^4.12.8 || ^5.0.0", + "@clerk/react": "^6.4.3", + "react": "^18.0.0 || ^19.0.0-0 || ^19.0.0" + }, + "peerDependenciesMeta": { + "@auth0/auth0-react": { + "optional": true + }, + "@clerk/clerk-react": { + "optional": true + }, + "@clerk/react": { + "optional": true + }, + "react": { + "optional": true + } + } + }, + "node_modules/esbuild": { + "version": "0.27.0", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.27.0.tgz", + "integrity": "sha512-jd0f4NHbD6cALCyGElNpGAOtWxSq46l9X/sWB0Nzd5er4Kz2YTm+Vl0qKFT9KUJvD8+fiO8AvoHhFvEatfVixA==", + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.27.0", + "@esbuild/android-arm": "0.27.0", + "@esbuild/android-arm64": "0.27.0", + "@esbuild/android-x64": "0.27.0", + "@esbuild/darwin-arm64": "0.27.0", + "@esbuild/darwin-x64": "0.27.0", + "@esbuild/freebsd-arm64": "0.27.0", + "@esbuild/freebsd-x64": "0.27.0", + "@esbuild/linux-arm": "0.27.0", + "@esbuild/linux-arm64": "0.27.0", + "@esbuild/linux-ia32": "0.27.0", + "@esbuild/linux-loong64": "0.27.0", + "@esbuild/linux-mips64el": "0.27.0", + "@esbuild/linux-ppc64": "0.27.0", + "@esbuild/linux-riscv64": "0.27.0", + "@esbuild/linux-s390x": "0.27.0", + "@esbuild/linux-x64": "0.27.0", + "@esbuild/netbsd-arm64": "0.27.0", + "@esbuild/netbsd-x64": "0.27.0", + "@esbuild/openbsd-arm64": "0.27.0", + "@esbuild/openbsd-x64": "0.27.0", + "@esbuild/openharmony-arm64": "0.27.0", + "@esbuild/sunos-x64": "0.27.0", + "@esbuild/win32-arm64": "0.27.0", + "@esbuild/win32-ia32": "0.27.0", + "@esbuild/win32-x64": "0.27.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/prettier": { + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.6.tgz", + "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/tsx": { + "version": "4.23.5", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.5.tgz", + "integrity": "sha512-rw55FUaqOoI7RvlQwLbhO4nSDApnQ4/CykPuiQ/EPvtrX3WA9Ig55jIt9VvbBJbzJuj12ueRu4PMZ2SxPVbihg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/tsx/node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/tsx/node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/ws": { + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.0.tgz", + "integrity": "sha512-Vsp28b7DRcimFQvrqu2Wek3z1iYxDCWqHYB8Qsnk/S4RfaCQzPGPyBNuVjJV3cd6UiKtUtp6sNM77gWvzcCH+g==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + } + } +} diff --git a/tools/stack-bench/tasks/team-chat/convex/tests/package.json b/tools/stack-bench/tasks/team-chat/convex/tests/package.json new file mode 100644 index 00000000000..67efe94a7d8 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/tests/package.json @@ -0,0 +1,13 @@ +{ + "name": "stack-bench-verifier-teamchat-convex", + "version": "0.0.1", + "private": true, + "type": "module", + "description": "Convex team-chat verifier: runs the shared behavioral scenario via the Convex client SDK.", + "dependencies": { + "convex": "^1.17.0" + }, + "devDependencies": { + "tsx": "^4.19.0" + } +} diff --git a/tools/stack-bench/tasks/team-chat/convex/tests/test.sh b/tools/stack-bench/tasks/team-chat/convex/tests/test.sh new file mode 100644 index 00000000000..3cdd5162868 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/convex/tests/test.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Thin launcher — Harbor's verifier entry point. Runs the shared team-chat +# behavioral scenario through the Convex adapter. +set -euo pipefail +cd "$(dirname "$0")" +npm install --no-audit --no-fund --silent +export RESTART_CMD="${RESTART_CMD:-/opt/stack-bench/backendctl restart}" +exec npx tsx harness/src/runTeamChat.ts "$(pwd)/adapter.ts" diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/environment/Dockerfile b/tools/stack-bench/tasks/team-chat/spacetimedb/environment/Dockerfile new file mode 100644 index 00000000000..be766e1d955 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/environment/Dockerfile @@ -0,0 +1,31 @@ +# Environment the agent works in: Rust + SpacetimeDB CLI + Node 22, with a local +# SpacetimeDB instance managed by /opt/stack-bench/backendctl (started at boot, +# restarted by the verifier's durability phase). +# Rust 1.93: the spacetimedb 2.7.x crates require rustc >= 1.93. +FROM rust:1.93-bookworm + +RUN apt-get update && apt-get install -y --no-install-recommends curl ca-certificates procps \ + && rm -rf /var/lib/apt/lists/* + +# Node 22 — supplies the global WebSocket the SpacetimeDB TS SDK uses. +RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \ + && apt-get install -y --no-install-recommends nodejs \ + && rm -rf /var/lib/apt/lists/* + +# wasm target for building SpacetimeDB modules. +RUN rustup target add wasm32-unknown-unknown + +# SpacetimeDB CLI. The committed module_bindings + pinned npm SDK are 2.5.x; +# if the installer's latest drifts incompatibly, pin here and regenerate. +RUN curl -sSf https://install.spacetimedb.com | sh -s -- --yes +ENV PATH="/root/.local/bin:/root/.local/share/spacetime/bin/current:${PATH}" + +COPY backendctl /opt/stack-bench/backendctl +RUN chmod +x /opt/stack-bench/backendctl && mkdir -p /var/lib/stdb +WORKDIR /app +EXPOSE 3000 + +# Boot the backend via backendctl, then exec Harbor's keepalive command so the +# agent and the SHARED-mode verifier share this container with the server up. +ENTRYPOINT ["/bin/sh", "-c", "/opt/stack-bench/backendctl start && exec \"$@\"", "sh"] +CMD ["sh", "-c", "sleep infinity"] diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/environment/backendctl b/tools/stack-bench/tasks/team-chat/spacetimedb/environment/backendctl new file mode 100644 index 00000000000..8e6d616da8a --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/environment/backendctl @@ -0,0 +1,52 @@ +#!/bin/sh +# Backend lifecycle control for the in-container SpacetimeDB instance. +# Used by the Dockerfile ENTRYPOINT at boot and by the verifier's durability +# phase (RESTART_CMD="/opt/stack-bench/backendctl restart"). Start and +# restart-after-kill go through the same code path so they behave identically. +set -eu + +DATA_DIR=/var/lib/stdb +LISTEN=0.0.0.0:3000 +LOG=/var/log/spacetime.log + +start() { + nohup spacetime start --data-dir "$DATA_DIR" --listen-addr "$LISTEN" --non-interactive >>"$LOG" 2>&1 & +} + +stop() { + # `spacetime start` execs the standalone server; kill both spellings. + pkill -f spacetimedb-standalone 2>/dev/null || true + pkill -f "spacetime start" 2>/dev/null || true + # Wait for the port to actually close. + i=0 + while curl -s -o /dev/null --max-time 1 http://127.0.0.1:3000 2>/dev/null; do + i=$((i + 1)) + [ "$i" -ge 30 ] && { echo "backendctl: server did not stop" >&2; exit 1; } + sleep 0.5 + done +} + +wait_ready() { + i=0 + # The server answers HTTP (404 on /) once ready; plain curl just checks the port. + until curl -s -o /dev/null --max-time 2 http://127.0.0.1:3000 2>/dev/null; do + i=$((i + 1)) + [ "$i" -ge 60 ] && { echo "backendctl: server did not become ready" >&2; exit 1; } + sleep 1 + done +} + +case "${1:-}" in + start) start ;; + stop) stop ;; + wait-ready) wait_ready ;; + restart) + stop + start + wait_ready + ;; + *) + echo "usage: backendctl {start|stop|restart|wait-ready}" >&2 + exit 2 + ;; +esac diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/instruction.md b/tools/stack-bench/tasks/team-chat/spacetimedb/instruction.md new file mode 100644 index 00000000000..fd1ec83fec7 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/instruction.md @@ -0,0 +1,193 @@ + + +# Task: team chat backend + +Build the backend for a team chat application (think a minimal Slack/Discord +server): named rooms with owners and members, real-time messaging with +editing and deletion, per-member unread counters, user presence, and a credit +system where users can tip each other. + +The application is graded **behaviorally** by an automated verifier that +drives the running backend through its real client SDK with multiple +concurrent clients. It checks functional correctness, transactional +correctness under concurrency, real-time delivery, durability across a +backend restart, and basic performance. Partial credit is awarded per check. +Grading does not inspect your source code — only observable behavior counts. + +## Data model (required behavior, not storage advice) + +- **User** — `username` (unique string), `status` (one of `"online"`, + `"away"`, `"offline"`), `balance` (integer credits). +- **Room** — `name` (unique string), `owner` (a username). +- **Membership** — which users are in which rooms, plus per-member + `last_read_seq` and `unread` counters (see Unread rules). +- **Message** — belongs to a room; has a server-assigned per-room sequence + number `seq`, a client-supplied id `client_msg_id`, `sender`, `text`, + `edited` flag, `deleted` flag. + +## Operations + +All operations validate their inputs and **fail with an error** when a rule +below is violated. "Fails" must be observable to the caller through the +backend's normal error mechanism (rejected mutation / failed reducer call). + +1. **register(username)** — Creates the user with `status = "online"` and + `balance = 100`. If the user already exists, the call **succeeds** and + leaves the existing user completely unchanged (idempotent; must NOT reset + balance or status). +2. **set_status(username, status)** — Sets presence. Fails for unknown users + or a status outside the three allowed values. +3. **create_room(username, room)** — Creates a room owned by `username`, who + automatically becomes a member (with `last_read_seq = 0`, `unread = 0`). + Fails for unknown users or if the room name already exists. +4. **join_room(username, room)** — Adds the user as a member with + `last_read_seq = 0` and `unread` equal to the number of messages already + in the room sent by other users. Fails for unknown user/room. If already + a member, succeeds without changing the existing membership state. +5. **leave_room(username, room)** — Removes the membership. Fails for + unknown user/room, if not a member, or if the user is the room's owner + (owners cannot leave). +6. **kick(actor, room, target)** — Removes `target`'s membership. Fails + unless `actor` is the room's owner; fails if `target` is the owner or not + a member. +7. **send_message(sender, room, text, client_msg_id)** — Appends a message. + Fails for: unknown user/room, sender not a member, empty `text`, `text` + longer than 4000 characters. **Idempotency:** if a message with the same + `client_msg_id` already exists in the room, the call **succeeds** without + creating a second message (safe retry). Otherwise the server assigns + `seq` (see Sequence rules), stores the message with `edited = false`, + `deleted = false`, and **atomically** increments `unread` by 1 for every + member of the room except the sender. +8. **edit_message(actor, room, client_msg_id, new_text)** — Replaces the + text and sets `edited = true`. Fails unless `actor` is the original + sender; fails if the message is deleted or `new_text` fails the same + validation as send. +9. **delete_message(actor, room, client_msg_id)** — Tombstones the message: + sets `deleted = true` and **clears `text` to the empty string**. Allowed + for the original sender or the room owner; fails for anyone else or if + already deleted. The original text must not be recoverable by any client + afterwards (late-joining clients must see the tombstone, never the text). +10. **mark_read(user, room, up_to_seq)** — Sets the member's + `last_read_seq = max(current, up_to_seq)` (it never decreases), then + recomputes `unread` per the Unread rules. Fails if not a member. +11. **tip(from_user, to_user, amount)** — Atomically transfers `amount` + credits. Fails for: unknown users, `from_user == to_user`, + `amount <= 0`, or `from_user`'s balance below `amount`. Balances must + never go negative and total credits must be conserved, including under + concurrent tips. + +## Sequence rules (transactional correctness) + +- `seq` is **per room**, assigned by the server, starting at 1 for the + room's first message and increasing by exactly 1 per stored message — + **no gaps, no duplicates**, even when many clients send concurrently. +- The counter must survive a backend restart: the first message stored + after a restart continues from the pre-restart maximum (durable counter — + an in-memory counter that resets and reuses sequence numbers fails). + +## Unread rules + +At any quiescent moment, for every member `u` of room `r`: +`unread(u, r) == count of messages m in r with m.seq > last_read_seq(u, r) and m.sender != u`. +Deleted (tombstoned) messages still count — they were sent. This invariant +must hold exactly, including after concurrent sends from multiple clients +(the per-message unread increments must be atomic with message insertion). + +## Real-time requirements + +Connected clients observe changes via **push** (the backend's real-time +subscription mechanism, not client polling): + +- A client subscribed to a room receives every existing message at + subscribe time (in `seq` order, reflecting current edited/deleted state) + and every subsequent message thereafter, exactly once, in `seq` order. +- Message **edits and deletions** are pushed to subscribers in real time. +- Membership changes (join / leave / kick / read-state updates) are pushed + to subscribers of the room's membership. +- User changes (status, balance) are pushed to subscribers of the user list. +- Subscriptions are **per room**: a client subscribed to room A must not + receive room B's messages. + +## Durability requirements + +All state — users, balances, rooms, memberships, read state, messages +(including edits and tombstones), the idempotency record of seen +`client_msg_id`s, and the per-room sequence counters — must survive a +backend process restart. The verifier will restart the backend mid-run, +reconnect fresh clients, and check every piece of state plus continued +real-time operation. + +## Performance requirements + +Modest but real: with a handful of clients, message delivery latency +(send-call to another subscribed client receiving the push) should be well +under 1.5 seconds at p95, and a burst of 120 messages from 3 concurrent +senders should be fully delivered to a subscriber within 45 seconds. A +correct implementation on this backend passes these comfortably; polling +loops with long intervals do not. + +## What is graded + +The verifier awards partial credit across four weighted groups: +correctness & transactions (0.40), real-time behavior (0.30), durability +across restart (0.20), performance (0.10). Every check is machine-verified +through the client SDK against the running backend. + +Your implementation MUST expose exactly the identifiers named in the +**Contract** section below so the grader can connect. + +## Contract (SpacetimeDB) + +A SpacetimeDB instance is already running in this container at +`http://127.0.0.1:3000` (WebSocket `ws://127.0.0.1:3000`). Build a SpacetimeDB +module (Rust recommended; the toolchain and `spacetime` CLI are installed) and +publish it as the database named **`teamchat`**: + + spacetime publish --server http://127.0.0.1:3000 -y -p teamchat + +The grader connects with the SpacetimeDB TypeScript SDK using bindings +generated from the schema below. Your module MUST define **exactly** these +public tables (same table names, column names, column types, in this order) +and reducers (same names and parameters). You may add extra indexes and +private tables, but not extra columns on these tables. + +Tables (all `public`): + + user: username: String (#[primary_key]), status: String, balance: i32 + room: name: String (#[primary_key]), owner: String, next_seq: u32 + member: id: u64 (#[primary_key] #[auto_inc]), room: String, user: String, + last_read_seq: u32, unread: u32 + message: id: u64 (#[primary_key] #[auto_inc]), room: String, seq: u32, + client_msg_id: String, sender: String, text: String, + edited: bool, deleted: bool, sent_at_micros: i64 + +Reducers (each returns `Result<(), String>`; errors are how failures are +reported to clients): + + register(username: String) + set_status(username: String, status: String) + create_room(username: String, room: String) + join_room(username: String, room: String) + leave_room(username: String, room: String) + kick(actor: String, room: String, target: String) + send_message(sender: String, room: String, text: String, client_msg_id: String) + edit_message(actor: String, room: String, client_msg_id: String, new_text: String) + delete_message(actor: String, room: String, client_msg_id: String) + mark_read(user: String, room: String, up_to_seq: u32) + tip(from_user: String, to_user: String, amount: i32) + +Notes: + +- The grader subscribes with SQL like `SELECT * FROM message WHERE room = '…'`, + so subscription filtering must work against these exact columns. +- SpacetimeDB reducers run as serializable transactions and the standalone + server persists to its commitlog; use `--data-dir`-backed state only (do not + cache application state in module globals — the verifier restarts the + server process and expects all state, including per-room `next_seq` counters + and `client_msg_id` dedupe behavior, to survive). +- Do not modify or stop the running server; publish your module to it. diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/solution/module/Cargo.lock b/tools/stack-bench/tasks/team-chat/spacetimedb/solution/module/Cargo.lock new file mode 100644 index 00000000000..31d0dd1d7c3 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/solution/module/Cargo.lock @@ -0,0 +1,806 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "anyhow" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" + +[[package]] +name = "approx" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0e60b75072ecd4168020818c0107f2857bb6c4e64252d8d3983f6263b40a5c3" +dependencies = [ + "num-traits", +] + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures 0.3.0", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + +[[package]] +name = "cc" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5add81bb678e6cb321aff7fa0dc7689ad82b112dbc032cea19f91d6b8e3582b9" +dependencies = [ + "find-msvc-tools", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "num-traits", +] + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "decorum" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "281759d3c8a14f5c3f0c49363be56810fcd7f910422f97f2db850c2920fde5cf" +dependencies = [ + "approx", + "num-traits", +] + +[[package]] +name = "derive_more" +version = "0.99.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6edb4b64a43d977b8e99788fe3a04d483834fba1215a7e02caa415b626497f7f" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 2.0.119", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "ethnum" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40404c3f5f511ec4da6fe866ddf6a717c309fdbb69fbbad7b0f3edab8f2e835f" +dependencies = [ + "serde", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "humantime" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "keccak" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb26cec98cce3a3d96cbb7bced3c4b16e3d13f27ec56dbd62cbc8f39cfb9d653" +dependencies = [ + "cpufeatures 0.2.17", +] + +[[package]] +name = "lean_string" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a262b6ae1dd9c2d3cf7977a816578b03bf8fb60b61545c395880f95eefc5b24" +dependencies = [ + "castaway", + "itoa", + "ryu", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nohash-hasher" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bf50223579dc7cdcfb3bfcacf7069ff68243f8c363f62ffa99cf000a6b9c451" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22f6172bdec972074665ed81ed53b71da00bfc44b65a753cfde883ec4c702a1a" +dependencies = [ + "libc", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + +[[package]] +name = "second-stack" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4904c83c6e51f1b9b08bfa5a86f35a51798e8307186e6f5513852210a219c0bb" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha3" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77fd7028345d415a4034cf8777cd4f8ab1851274233b45f84e3d955502d93874" +dependencies = [ + "digest", + "keccak", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "spacetimedb" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcef3a8fd4e0f6854a30e36cfc74bee6d39b93ffc8b7268bbe497f01bef29872" +dependencies = [ + "anyhow", + "bytemuck", + "bytes", + "derive_more", + "getrandom 0.2.17", + "http", + "log", + "rand 0.8.7", + "scoped-tls", + "serde_json", + "spacetimedb-bindings-macro", + "spacetimedb-bindings-sys", + "spacetimedb-lib", + "spacetimedb-primitives", + "spacetimedb-query-builder", +] + +[[package]] +name = "spacetimedb-bindings-macro" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0dc702a13b6e59650e89eadd16662d9ced6465ed69ee1d81149998cb6925992" +dependencies = [ + "heck 0.4.1", + "humantime", + "proc-macro2", + "quote", + "spacetimedb-primitives", + "syn 2.0.119", +] + +[[package]] +name = "spacetimedb-bindings-sys" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b66468bd6aebb1677394bbd0728b5a61af3ca569dbd6f8cf6ed062a3d8a26a7" +dependencies = [ + "spacetimedb-primitives", +] + +[[package]] +name = "spacetimedb-lib" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33cbe183b0c344ebbcebc2b928b424f125bb0e46c94258b9674396e64617b980" +dependencies = [ + "anyhow", + "bitflags", + "blake3", + "bytes", + "chrono", + "derive_more", + "enum-as-inner", + "hex", + "itertools", + "log", + "spacetimedb-bindings-macro", + "spacetimedb-primitives", + "spacetimedb-sats", + "thiserror", +] + +[[package]] +name = "spacetimedb-primitives" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49b8567ae448b4ccc39f75be5813681725eaaeb7c566ab73a2b1ca91427df0ee" +dependencies = [ + "bitflags", + "either", + "enum-as-inner", + "itertools", + "nohash-hasher", +] + +[[package]] +name = "spacetimedb-query-builder" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b09e19e3c0ba651dba2208b03f0266df7038a79d741e2262b3b5a83d4dc6c461" +dependencies = [ + "spacetimedb-lib", +] + +[[package]] +name = "spacetimedb-sats" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca26c1f95c8dbfb29621ea5709fcdc882a49eb6597f11e458cc7f6917bfd364b" +dependencies = [ + "anyhow", + "arrayvec", + "bitflags", + "bytemuck", + "bytes", + "chrono", + "decorum", + "derive_more", + "enum-as-inner", + "ethnum", + "hex", + "itertools", + "lean_string", + "rand 0.9.5", + "second-stack", + "sha3", + "smallvec", + "spacetimedb-bindings-macro", + "spacetimedb-primitives", + "thiserror", + "uuid", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "teamchat-module" +version = "0.1.0" +dependencies = [ + "spacetimedb", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.3", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/solution/module/Cargo.toml b/tools/stack-bench/tasks/team-chat/spacetimedb/solution/module/Cargo.toml new file mode 100644 index 00000000000..f35bc597e11 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/solution/module/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "teamchat-module" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +# Keep this module out of the repo's cargo workspace (it is its own project). +[workspace] + +[dependencies] +# Match the major version of the spacetime CLI installed in the environment image. +spacetimedb = "2" diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/solution/module/src/lib.rs b/tools/stack-bench/tasks/team-chat/spacetimedb/solution/module/src/lib.rs new file mode 100644 index 00000000000..0a43bfef868 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/solution/module/src/lib.rs @@ -0,0 +1,343 @@ +//! Reference (oracle) SpacetimeDB module for the stack-bench team-chat task. +//! +//! Implements the full team-chat spec: users/presence/credits, rooms with +//! owners, membership with server-maintained unread counters, messages with +//! per-room gapless seq, idempotent sends, edit/tombstone-delete, and atomic +//! tips. SpacetimeDB reducers are serializable transactions, so the +//! concurrency checks (gapless seq, unread invariant, tip conservation) hold +//! by construction; the standalone server's commitlog provides restart +//! durability. + +use spacetimedb::{reducer, table, ReducerContext, Table}; + +const STATUSES: [&str; 3] = ["online", "away", "offline"]; +const MAX_TEXT: usize = 4000; + +#[table(accessor = user, public)] +pub struct User { + #[primary_key] + pub username: String, + pub status: String, + pub balance: i32, +} + +#[table(accessor = room, public)] +pub struct Room { + #[primary_key] + pub name: String, + pub owner: String, + pub next_seq: u32, +} + +#[table(accessor = member, public, + index(accessor = by_room, btree(columns = [room])), + index(accessor = by_room_user, btree(columns = [room, user])))] +pub struct Member { + #[primary_key] + #[auto_inc] + pub id: u64, + pub room: String, + pub user: String, + pub last_read_seq: u32, + pub unread: u32, +} + +#[table(accessor = message, public, + index(accessor = by_room, btree(columns = [room])), + index(accessor = by_room_client, btree(columns = [room, client_msg_id])))] +pub struct Message { + #[primary_key] + #[auto_inc] + pub id: u64, + pub room: String, + pub seq: u32, + pub client_msg_id: String, + pub sender: String, + pub text: String, + pub edited: bool, + pub deleted: bool, + pub sent_at_micros: i64, +} + +// ---- helpers ---- + +fn get_user(ctx: &ReducerContext, username: &str) -> Result { + ctx.db + .user() + .username() + .find(username.to_owned()) + .ok_or_else(|| format!("unknown user: {username}")) +} + +fn get_room(ctx: &ReducerContext, name: &str) -> Result { + ctx.db + .room() + .name() + .find(name.to_owned()) + .ok_or_else(|| format!("unknown room: {name}")) +} + +fn find_member(ctx: &ReducerContext, room: &str, user: &str) -> Option { + ctx.db + .member() + .by_room_user() + .filter((room, user)) + .next() +} + +fn validate_text(text: &str) -> Result<(), String> { + if text.is_empty() { + return Err("text must not be empty".into()); + } + if text.chars().count() > MAX_TEXT { + return Err(format!("text exceeds {MAX_TEXT} characters")); + } + Ok(()) +} + +// ---- reducers ---- + +#[reducer] +pub fn register(ctx: &ReducerContext, username: String) -> Result<(), String> { + if username.is_empty() { + return Err("username must not be empty".into()); + } + // Idempotent: existing users are left completely unchanged. + if ctx.db.user().username().find(&username).is_none() { + ctx.db.user().insert(User { + username, + status: "online".into(), + balance: 100, + }); + } + Ok(()) +} + +#[reducer] +pub fn set_status(ctx: &ReducerContext, username: String, status: String) -> Result<(), String> { + if !STATUSES.contains(&status.as_str()) { + return Err(format!("invalid status: {status}")); + } + let mut user = get_user(ctx, &username)?; + user.status = status; + ctx.db.user().username().update(user); + Ok(()) +} + +#[reducer] +pub fn create_room(ctx: &ReducerContext, username: String, room: String) -> Result<(), String> { + get_user(ctx, &username)?; + if room.is_empty() { + return Err("room name must not be empty".into()); + } + if ctx.db.room().name().find(&room).is_some() { + return Err(format!("room already exists: {room}")); + } + ctx.db.room().insert(Room { + name: room.clone(), + owner: username.clone(), + next_seq: 1, + }); + ctx.db.member().insert(Member { + id: 0, + room, + user: username, + last_read_seq: 0, + unread: 0, + }); + Ok(()) +} + +#[reducer] +pub fn join_room(ctx: &ReducerContext, username: String, room: String) -> Result<(), String> { + get_user(ctx, &username)?; + get_room(ctx, &room)?; + // Idempotent: existing membership state is preserved. + if find_member(ctx, &room, &username).is_some() { + return Ok(()); + } + let unread = ctx + .db + .message() + .by_room() + .filter(&room) + .filter(|m| m.sender != username) + .count() as u32; + ctx.db.member().insert(Member { + id: 0, + room, + user: username, + last_read_seq: 0, + unread, + }); + Ok(()) +} + +#[reducer] +pub fn leave_room(ctx: &ReducerContext, username: String, room: String) -> Result<(), String> { + get_user(ctx, &username)?; + let r = get_room(ctx, &room)?; + if r.owner == username { + return Err("the room owner cannot leave".into()); + } + let member = find_member(ctx, &room, &username).ok_or("not a member")?; + ctx.db.member().id().delete(member.id); + Ok(()) +} + +#[reducer] +pub fn kick(ctx: &ReducerContext, actor: String, room: String, target: String) -> Result<(), String> { + get_user(ctx, &actor)?; + let r = get_room(ctx, &room)?; + if r.owner != actor { + return Err("only the room owner can kick".into()); + } + if target == r.owner { + return Err("cannot kick the room owner".into()); + } + let member = find_member(ctx, &room, &target).ok_or("target is not a member")?; + ctx.db.member().id().delete(member.id); + Ok(()) +} + +#[reducer] +pub fn send_message( + ctx: &ReducerContext, + sender: String, + room: String, + text: String, + client_msg_id: String, +) -> Result<(), String> { + get_user(ctx, &sender)?; + let mut r = get_room(ctx, &room)?; + find_member(ctx, &room, &sender).ok_or("sender is not a member")?; + validate_text(&text)?; + // Idempotent retry: same client_msg_id in this room -> success, no new row. + if ctx + .db + .message() + .by_room_client() + .filter((&room, &client_msg_id)) + .next() + .is_some() + { + return Ok(()); + } + let seq = r.next_seq; + r.next_seq += 1; + ctx.db.room().name().update(r); + ctx.db.message().insert(Message { + id: 0, + room: room.clone(), + seq, + client_msg_id, + sender: sender.clone(), + text, + edited: false, + deleted: false, + sent_at_micros: ctx.timestamp.to_micros_since_unix_epoch(), + }); + // Atomic with the insert (same reducer transaction): bump unread for + // every member except the sender. + let members: Vec = ctx.db.member().by_room().filter(&room).collect(); + for mut m in members { + if m.user != sender { + m.unread += 1; + ctx.db.member().id().update(m); + } + } + Ok(()) +} + +fn find_message(ctx: &ReducerContext, room: &str, client_msg_id: &str) -> Result { + ctx.db + .message() + .by_room_client() + .filter((room, client_msg_id)) + .next() + .ok_or_else(|| format!("no message {client_msg_id} in {room}")) +} + +#[reducer] +pub fn edit_message( + ctx: &ReducerContext, + actor: String, + room: String, + client_msg_id: String, + new_text: String, +) -> Result<(), String> { + get_user(ctx, &actor)?; + get_room(ctx, &room)?; + let mut msg = find_message(ctx, &room, &client_msg_id)?; + if msg.sender != actor { + return Err("only the original sender can edit".into()); + } + if msg.deleted { + return Err("cannot edit a deleted message".into()); + } + validate_text(&new_text)?; + msg.text = new_text; + msg.edited = true; + ctx.db.message().id().update(msg); + Ok(()) +} + +#[reducer] +pub fn delete_message( + ctx: &ReducerContext, + actor: String, + room: String, + client_msg_id: String, +) -> Result<(), String> { + get_user(ctx, &actor)?; + let r = get_room(ctx, &room)?; + let mut msg = find_message(ctx, &room, &client_msg_id)?; + if msg.sender != actor && r.owner != actor { + return Err("only the sender or the room owner can delete".into()); + } + if msg.deleted { + return Err("message is already deleted".into()); + } + msg.deleted = true; + msg.text = String::new(); + ctx.db.message().id().update(msg); + Ok(()) +} + +#[reducer] +pub fn mark_read(ctx: &ReducerContext, user: String, room: String, up_to_seq: u32) -> Result<(), String> { + get_user(ctx, &user)?; + get_room(ctx, &room)?; + let mut member = find_member(ctx, &room, &user).ok_or("not a member")?; + // Monotone: last_read_seq never decreases. + member.last_read_seq = member.last_read_seq.max(up_to_seq); + member.unread = ctx + .db + .message() + .by_room() + .filter(&room) + .filter(|m| m.seq > member.last_read_seq && m.sender != user) + .count() as u32; + ctx.db.member().id().update(member); + Ok(()) +} + +#[reducer] +pub fn tip(ctx: &ReducerContext, from_user: String, to_user: String, amount: i32) -> Result<(), String> { + if from_user == to_user { + return Err("cannot tip yourself".into()); + } + if amount <= 0 { + return Err("amount must be positive".into()); + } + let mut from = get_user(ctx, &from_user)?; + let mut to = get_user(ctx, &to_user)?; + if from.balance < amount { + return Err("insufficient balance".into()); + } + from.balance -= amount; + to.balance += amount; + ctx.db.user().username().update(from); + ctx.db.user().username().update(to); + Ok(()) +} diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/solution/solve.sh b/tools/stack-bench/tasks/team-chat/spacetimedb/solution/solve.sh new file mode 100644 index 00000000000..b83657d8e08 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/solution/solve.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +# Thin launcher — Harbor's oracle entry point. Publishes the reference module. +# Backend readiness is gated by the task.toml healthcheck. +set -euo pipefail +exec spacetime publish --server "${STDB_SERVER:-http://127.0.0.1:3000}" -y \ + -p "$(dirname "$0")/module" teamchat diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/task.toml b/tools/stack-bench/tasks/team-chat/spacetimedb/task.toml new file mode 100644 index 00000000000..af089337dcb --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/task.toml @@ -0,0 +1,42 @@ +schema_version = "1.2" +artifacts = [] + +[task] +name = "clockwork/team-chat-spacetimedb" +description = "Build a full team-chat backend (rooms, presence, unread counters, edits/tombstones, idempotent sends, gapless per-room seq, atomic tips) on SpacetimeDB; graded behaviorally incl. concurrency, restart durability, and latency." +keywords = ["realtime", "chat", "spacetimedb", "cross-backend", "stack-bench", "durability", "acid"] +[[task.authors]] +name = "Clockwork Labs" +email = "" + +[metadata] +# stack-bench-specific: which backend this task variant exercises. +backend = "spacetimedb" +tags = ["spacetimedb"] +category = "team-chat" + +[verifier] +# The scenario runs ~44 checks including a backend restart and perf probes. +timeout_sec = 1200.0 +# SHARED mode (default with no [verifier.environment]): the verifier runs +# inside the agent's container and reaches the app + backendctl directly. + +[agent] +timeout_sec = 3600.0 + +[environment] +build_timeout_sec = 1800.0 +os = "linux" +cpus = 2 +memory_mb = 4096 +storage_mb = 12288 +gpus = 0 +allow_internet = true + +# Gate the agent on the in-container SpacetimeDB instance being ready. +[environment.healthcheck] +command = "curl -s -o /dev/null http://127.0.0.1:3000" +interval_sec = 3.0 +timeout_sec = 10.0 +start_period_sec = 5.0 +retries = 30 diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/adapter.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/adapter.ts new file mode 100644 index 00000000000..660900ffc7a --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/adapter.ts @@ -0,0 +1,211 @@ +// SpacetimeDB adapter for the stack-bench team-chat task. +// +// Implements the harness AppClient contract against the running `teamchat` +// database via the SpacetimeDB TypeScript SDK (2.5.x) + generated bindings. +// +// - Mutations use the SDK's promise-based reducer calls: they resolve when the +// reducer commits and reject with the reducer's error message when it fails, +// which is exactly the contract's accept/reject semantics. +// - connect() subscribes to user/room/member; message subscriptions are +// created per-room with a server-side WHERE filter, so cross-room isolation +// is graded against real backend routing, not a client-side filter. +// - History-then-live ordering: initial-sync message inserts are buffered +// until the subscription is applied, sorted by seq, then emitted. + +import { DbConnection } from "./module_bindings/index.js"; +import type { + AppClient, + MemberRecord, + MessageRecord, + MemberEventHandlers, + RoomEventHandlers, + UserEventHandlers, + UserRecord, +} from "./harness/src/appClient.js"; + +const URI = process.env.STDB_URI ?? "ws://127.0.0.1:3000"; +const DB = process.env.STDB_DB ?? "teamchat"; + +type MessageRow = { + seq: number; + clientMsgId: string; + sender: string; + text: string; + edited: boolean; + deleted: boolean; + room: string; +}; + +const msgRec = (r: MessageRow): MessageRecord => ({ + seq: Number(r.seq), + clientMsgId: r.clientMsgId, + sender: r.sender, + text: r.text, + edited: r.edited, + deleted: r.deleted, +}); + +class SpacetimeAppClient implements AppClient { + private conn: any; + private roomSubs = new Set(); + + async connect(): Promise { + await new Promise((resolve, reject) => { + this.conn = DbConnection.builder() + .withUri(URI) + .withDatabaseName(DB) + .onConnect(() => resolve()) + .onConnectError((_ctx: any, err: any) => reject(err ?? new Error("connect error"))) + .build(); + }); + // Base subscriptions (small tables). Message subs are per-room. + await this.subscribeSql(["SELECT * FROM user", "SELECT * FROM room", "SELECT * FROM member"]); + } + + private subscribeSql(queries: string[]): Promise { + return new Promise((resolve, reject) => { + this.conn + .subscriptionBuilder() + .onApplied(() => resolve()) + .onError((_ctx: any, err: any) => reject(err ?? new Error("subscription error"))) + .subscribe(queries); + }); + } + + /** Server-side filtered subscription for one room's messages. */ + private async ensureRoomSub(room: string): Promise { + if (this.roomSubs.has(room)) return; + this.roomSubs.add(room); + await this.subscribeSql([`SELECT * FROM message WHERE room = '${room.replace(/'/g, "''")}'`]); + } + + async close(): Promise { + try { + this.conn?.disconnect(); + } catch { + /* ignore */ + } + } + + // ---- mutations ---- + register(username: string) { + return this.conn.reducers.register({ username }); + } + setStatus(username: string, status: string) { + return this.conn.reducers.setStatus({ username, status }); + } + createRoom(username: string, room: string) { + return this.conn.reducers.createRoom({ username, room }); + } + joinRoom(username: string, room: string) { + return this.conn.reducers.joinRoom({ username, room }); + } + leaveRoom(username: string, room: string) { + return this.conn.reducers.leaveRoom({ username, room }); + } + kick(actor: string, room: string, target: string) { + return this.conn.reducers.kick({ actor, room, target }); + } + sendMessage(sender: string, room: string, text: string, clientMsgId: string) { + return this.conn.reducers.sendMessage({ sender, room, text, clientMsgId }); + } + editMessage(actor: string, room: string, clientMsgId: string, newText: string) { + return this.conn.reducers.editMessage({ actor, room, clientMsgId, newText }); + } + deleteMessage(actor: string, room: string, clientMsgId: string) { + return this.conn.reducers.deleteMessage({ actor, room, clientMsgId }); + } + markRead(user: string, room: string, upToSeq: number) { + return this.conn.reducers.markRead({ user, room, upToSeq }); + } + tip(fromUser: string, toUser: string, amount: number) { + return this.conn.reducers.tip({ fromUser, toUser, amount }); + } + + // ---- subscriptions ---- + async subscribeRoom(room: string, handlers: RoomEventHandlers): Promise { + // Register callbacks first, buffering until the room subscription applies; + // then emit buffered history sorted by seq, then live events directly. + let applied = false; + const buffer: MessageRow[] = []; + const emit = (row: MessageRow) => { + if (row.room !== room) return; + if (applied) handlers.onMessage(msgRec(row)); + else buffer.push(row); + }; + this.conn.db.message.onInsert((_ctx: any, row: MessageRow) => emit(row)); + this.conn.db.message.onUpdate((_ctx: any, _old: MessageRow, row: MessageRow) => emit(row)); + await this.ensureRoomSub(room); + buffer.sort((a, b) => Number(a.seq) - Number(b.seq)); + for (const row of buffer) handlers.onMessage(msgRec(row)); + applied = true; + } + + async subscribeUsers(handlers: UserEventHandlers): Promise { + const emit = (row: any) => + handlers.onUser({ username: row.username, status: row.status, balance: Number(row.balance) }); + this.conn.db.user.onInsert((_ctx: any, row: any) => emit(row)); + this.conn.db.user.onUpdate((_ctx: any, _old: any, row: any) => emit(row)); + for (const row of this.conn.db.user.iter()) emit(row); + } + + async subscribeMembers(room: string, handlers: MemberEventHandlers): Promise { + const rec = (row: any): MemberRecord => ({ + user: row.user, + lastReadSeq: Number(row.lastReadSeq), + unread: Number(row.unread), + }); + this.conn.db.member.onInsert((_ctx: any, row: any) => { + if (row.room === room) handlers.onMember(rec(row)); + }); + this.conn.db.member.onUpdate((_ctx: any, _old: any, row: any) => { + if (row.room === room) handlers.onMember(rec(row)); + }); + this.conn.db.member.onDelete((_ctx: any, row: any) => { + if (row.room === room) handlers.onMemberRemoved?.(row.user); + }); + for (const row of this.conn.db.member.iter()) { + if (row.room === room) handlers.onMember(rec(row)); + } + } + + // ---- snapshots ---- + async getUser(username: string): Promise { + for (const row of this.conn.db.user.iter()) { + if (row.username === username) { + return { username: row.username, status: row.status, balance: Number(row.balance) }; + } + } + return null; + } + + async getRoomOwner(room: string): Promise { + for (const row of this.conn.db.room.iter()) { + if (row.name === room) return row.owner; + } + return null; + } + + async getMembers(room: string): Promise { + const out: MemberRecord[] = []; + for (const row of this.conn.db.member.iter()) { + if (row.room === room) { + out.push({ user: row.user, lastReadSeq: Number(row.lastReadSeq), unread: Number(row.unread) }); + } + } + return out.sort((a, b) => a.user.localeCompare(b.user)); + } + + async getMessages(room: string): Promise { + await this.ensureRoomSub(room); + const out: MessageRecord[] = []; + for (const row of this.conn.db.message.iter()) { + if (row.room === room) out.push(msgRec(row)); + } + return out.sort((a, b) => a.seq - b.seq); + } +} + +export default function makeClient(): AppClient { + return new SpacetimeAppClient(); +} diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/create_room_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/create_room_reducer.ts new file mode 100644 index 00000000000..8c0b8dac0af --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/create_room_reducer.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + username: __t.string(), + room: __t.string(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/delete_message_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/delete_message_reducer.ts new file mode 100644 index 00000000000..3f32b09b277 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/delete_message_reducer.ts @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + actor: __t.string(), + room: __t.string(), + clientMsgId: __t.string(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/edit_message_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/edit_message_reducer.ts new file mode 100644 index 00000000000..cdd5d5d21b3 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/edit_message_reducer.ts @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + actor: __t.string(), + room: __t.string(), + clientMsgId: __t.string(), + newText: __t.string(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/index.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/index.ts new file mode 100644 index 00000000000..3a0fe35b187 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/index.ts @@ -0,0 +1,192 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +// This was generated using spacetimedb cli version 2.5.0 (commit ca16958ef0a5f8c816700d2255a0b20ecacff901). + +/* eslint-disable */ +/* tslint:disable */ +import { + DbConnectionBuilder as __DbConnectionBuilder, + DbConnectionImpl as __DbConnectionImpl, + SubscriptionBuilderImpl as __SubscriptionBuilderImpl, + TypeBuilder as __TypeBuilder, + Uuid as __Uuid, + convertToAccessorMap as __convertToAccessorMap, + makeQueryBuilder as __makeQueryBuilder, + procedureSchema as __procedureSchema, + procedures as __procedures, + reducerSchema as __reducerSchema, + reducers as __reducers, + schema as __schema, + t as __t, + table as __table, + type AlgebraicTypeType as __AlgebraicTypeType, + type DbConnectionConfig as __DbConnectionConfig, + type ErrorContextInterface as __ErrorContextInterface, + type Event as __Event, + type EventContextInterface as __EventContextInterface, + type Infer as __Infer, + type QueryBuilder as __QueryBuilder, + type ReducerEventContextInterface as __ReducerEventContextInterface, + type RemoteModule as __RemoteModule, + type SubscriptionEventContextInterface as __SubscriptionEventContextInterface, + type SubscriptionHandleImpl as __SubscriptionHandleImpl, +} from "spacetimedb"; + +// Import all reducer arg schemas +import CreateRoomReducer from "./create_room_reducer"; +import DeleteMessageReducer from "./delete_message_reducer"; +import EditMessageReducer from "./edit_message_reducer"; +import JoinRoomReducer from "./join_room_reducer"; +import KickReducer from "./kick_reducer"; +import LeaveRoomReducer from "./leave_room_reducer"; +import MarkReadReducer from "./mark_read_reducer"; +import RegisterReducer from "./register_reducer"; +import SendMessageReducer from "./send_message_reducer"; +import SetStatusReducer from "./set_status_reducer"; +import TipReducer from "./tip_reducer"; + +// Import all procedure arg schemas + +// Import all table schema definitions +import MemberRow from "./member_table"; +import MessageRow from "./message_table"; +import RoomRow from "./room_table"; +import UserRow from "./user_table"; + +/** Type-only namespace exports for generated type groups. */ + +/** The schema information for all tables in this module. This is defined the same was as the tables would have been defined in the server. */ +const tablesSchema = __schema({ + member: __table({ + name: 'member', + indexes: [ + { accessor: 'id', name: 'member_id_idx_btree', algorithm: 'btree', columns: [ + 'id', + ] }, + { accessor: 'by_room', name: 'member_room_idx_btree', algorithm: 'btree', columns: [ + 'room', + ] }, + { accessor: 'by_room_user', name: 'member_room_user_idx_btree', algorithm: 'btree', columns: [ + 'room', + 'user', + ] }, + ], + constraints: [ + { name: 'member_id_key', constraint: 'unique', columns: ['id'] }, + ], + }, MemberRow), + message: __table({ + name: 'message', + indexes: [ + { accessor: 'id', name: 'message_id_idx_btree', algorithm: 'btree', columns: [ + 'id', + ] }, + { accessor: 'by_room_client', name: 'message_room_client_msg_id_idx_btree', algorithm: 'btree', columns: [ + 'room', + 'clientMsgId', + ] }, + { accessor: 'by_room', name: 'message_room_idx_btree', algorithm: 'btree', columns: [ + 'room', + ] }, + ], + constraints: [ + { name: 'message_id_key', constraint: 'unique', columns: ['id'] }, + ], + }, MessageRow), + room: __table({ + name: 'room', + indexes: [ + { accessor: 'name', name: 'room_name_idx_btree', algorithm: 'btree', columns: [ + 'name', + ] }, + ], + constraints: [ + { name: 'room_name_key', constraint: 'unique', columns: ['name'] }, + ], + }, RoomRow), + user: __table({ + name: 'user', + indexes: [ + { accessor: 'username', name: 'user_username_idx_btree', algorithm: 'btree', columns: [ + 'username', + ] }, + ], + constraints: [ + { name: 'user_username_key', constraint: 'unique', columns: ['username'] }, + ], + }, UserRow), +}); + +/** The schema information for all reducers in this module. This is defined the same way as the reducers would have been defined in the server, except the body of the reducer is omitted in code generation. */ +const reducersSchema = __reducers( + __reducerSchema("create_room", CreateRoomReducer), + __reducerSchema("delete_message", DeleteMessageReducer), + __reducerSchema("edit_message", EditMessageReducer), + __reducerSchema("join_room", JoinRoomReducer), + __reducerSchema("kick", KickReducer), + __reducerSchema("leave_room", LeaveRoomReducer), + __reducerSchema("mark_read", MarkReadReducer), + __reducerSchema("register", RegisterReducer), + __reducerSchema("send_message", SendMessageReducer), + __reducerSchema("set_status", SetStatusReducer), + __reducerSchema("tip", TipReducer), +); + +/** The schema information for all procedures in this module. This is defined the same way as the procedures would have been defined in the server. */ +const proceduresSchema = __procedures( +); + +/** The remote SpacetimeDB module schema, both runtime and type information. */ +const REMOTE_MODULE = { + versionInfo: { + cliVersion: "2.5.0" as const, + }, + tables: tablesSchema.schemaType.tables, + reducers: reducersSchema.reducersType.reducers, + ...proceduresSchema, +} satisfies __RemoteModule< + typeof tablesSchema.schemaType, + typeof reducersSchema.reducersType, + typeof proceduresSchema +>; + +/** The tables available in this remote SpacetimeDB module. Each table reference doubles as a query builder. */ +export const tables: __QueryBuilder = __makeQueryBuilder(tablesSchema.schemaType); + +/** The reducers available in this remote SpacetimeDB module. */ +export const reducers = __convertToAccessorMap(reducersSchema.reducersType.reducers); + +/** The procedures available in this remote SpacetimeDB module. */ +export const procedures = __convertToAccessorMap(proceduresSchema.procedures); + +/** The context type returned in callbacks for all possible events. */ +export type EventContext = __EventContextInterface; +/** The context type returned in callbacks for reducer events. */ +export type ReducerEventContext = __ReducerEventContextInterface; +/** The context type returned in callbacks for subscription events. */ +export type SubscriptionEventContext = __SubscriptionEventContextInterface; +/** The context type returned in callbacks for error events. */ +export type ErrorContext = __ErrorContextInterface; +/** The subscription handle type to manage active subscriptions created from a {@link SubscriptionBuilder}. */ +export type SubscriptionHandle = __SubscriptionHandleImpl; + +/** Builder class to configure a new subscription to the remote SpacetimeDB instance. */ +export class SubscriptionBuilder extends __SubscriptionBuilderImpl {} + +/** Builder class to configure a new database connection to the remote SpacetimeDB instance. */ +export class DbConnectionBuilder extends __DbConnectionBuilder {} + +/** The typed database connection to manage connections to the remote SpacetimeDB instance. This class has type information specific to the generated module. */ +export class DbConnection extends __DbConnectionImpl { + /** Creates a new {@link DbConnectionBuilder} to configure and connect to the remote SpacetimeDB instance. */ + static builder = (): DbConnectionBuilder => { + return new DbConnectionBuilder(REMOTE_MODULE, (config: __DbConnectionConfig) => new DbConnection(config)); + }; + + /** Creates a new {@link SubscriptionBuilder} to configure a subscription to the remote SpacetimeDB instance. */ + override subscriptionBuilder = (): SubscriptionBuilder => { + return new SubscriptionBuilder(this); + }; +} + diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/join_room_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/join_room_reducer.ts new file mode 100644 index 00000000000..8c0b8dac0af --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/join_room_reducer.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + username: __t.string(), + room: __t.string(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/kick_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/kick_reducer.ts new file mode 100644 index 00000000000..ef376a04a12 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/kick_reducer.ts @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + actor: __t.string(), + room: __t.string(), + target: __t.string(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/leave_room_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/leave_room_reducer.ts new file mode 100644 index 00000000000..8c0b8dac0af --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/leave_room_reducer.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + username: __t.string(), + room: __t.string(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/mark_read_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/mark_read_reducer.ts new file mode 100644 index 00000000000..ae57e2509dd --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/mark_read_reducer.ts @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + user: __t.string(), + room: __t.string(), + upToSeq: __t.u32(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/member_table.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/member_table.ts new file mode 100644 index 00000000000..9131a1e48f7 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/member_table.ts @@ -0,0 +1,19 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64().primaryKey(), + room: __t.string(), + user: __t.string(), + lastReadSeq: __t.u32().name("last_read_seq"), + unread: __t.u32(), +}); diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/message_table.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/message_table.ts new file mode 100644 index 00000000000..55798cdc0c9 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/message_table.ts @@ -0,0 +1,23 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + id: __t.u64().primaryKey(), + room: __t.string(), + seq: __t.u32(), + clientMsgId: __t.string().name("client_msg_id"), + sender: __t.string(), + text: __t.string(), + edited: __t.bool(), + deleted: __t.bool(), + sentAtMicros: __t.i64().name("sent_at_micros"), +}); diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/register_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/register_reducer.ts new file mode 100644 index 00000000000..c79d2c546db --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/register_reducer.ts @@ -0,0 +1,15 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + username: __t.string(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/room_table.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/room_table.ts new file mode 100644 index 00000000000..c49f695c2c0 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/room_table.ts @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + name: __t.string().primaryKey(), + owner: __t.string(), + nextSeq: __t.u32().name("next_seq"), +}); diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/send_message_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/send_message_reducer.ts new file mode 100644 index 00000000000..da990eef275 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/send_message_reducer.ts @@ -0,0 +1,18 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + sender: __t.string(), + room: __t.string(), + text: __t.string(), + clientMsgId: __t.string(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/set_status_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/set_status_reducer.ts new file mode 100644 index 00000000000..7ca6d5bc82c --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/set_status_reducer.ts @@ -0,0 +1,16 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + username: __t.string(), + status: __t.string(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/tip_reducer.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/tip_reducer.ts new file mode 100644 index 00000000000..6ffea02b16c --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/tip_reducer.ts @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default { + fromUser: __t.string(), + toUser: __t.string(), + amount: __t.i32(), +}; diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/types.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/types.ts new file mode 100644 index 00000000000..1981a020c30 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/types.ts @@ -0,0 +1,48 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export const Member = __t.object("Member", { + id: __t.u64(), + room: __t.string(), + user: __t.string(), + lastReadSeq: __t.u32(), + unread: __t.u32(), +}); +export type Member = __Infer; + +export const Message = __t.object("Message", { + id: __t.u64(), + room: __t.string(), + seq: __t.u32(), + clientMsgId: __t.string(), + sender: __t.string(), + text: __t.string(), + edited: __t.bool(), + deleted: __t.bool(), + sentAtMicros: __t.i64(), +}); +export type Message = __Infer; + +export const Room = __t.object("Room", { + name: __t.string(), + owner: __t.string(), + nextSeq: __t.u32(), +}); +export type Room = __Infer; + +export const User = __t.object("User", { + username: __t.string(), + status: __t.string(), + balance: __t.i32(), +}); +export type User = __Infer; + diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/types/procedures.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/types/procedures.ts new file mode 100644 index 00000000000..d5ac825c9ab --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/types/procedures.ts @@ -0,0 +1,10 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { type Infer as __Infer } from "spacetimedb"; + +// Import all procedure arg schemas + + diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/types/reducers.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/types/reducers.ts new file mode 100644 index 00000000000..32529916dcf --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/types/reducers.ts @@ -0,0 +1,32 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { type Infer as __Infer } from "spacetimedb"; + +// Import all reducer arg schemas +import CreateRoomReducer from "../create_room_reducer"; +import DeleteMessageReducer from "../delete_message_reducer"; +import EditMessageReducer from "../edit_message_reducer"; +import JoinRoomReducer from "../join_room_reducer"; +import KickReducer from "../kick_reducer"; +import LeaveRoomReducer from "../leave_room_reducer"; +import MarkReadReducer from "../mark_read_reducer"; +import RegisterReducer from "../register_reducer"; +import SendMessageReducer from "../send_message_reducer"; +import SetStatusReducer from "../set_status_reducer"; +import TipReducer from "../tip_reducer"; + +export type CreateRoomParams = __Infer; +export type DeleteMessageParams = __Infer; +export type EditMessageParams = __Infer; +export type JoinRoomParams = __Infer; +export type KickParams = __Infer; +export type LeaveRoomParams = __Infer; +export type MarkReadParams = __Infer; +export type RegisterParams = __Infer; +export type SendMessageParams = __Infer; +export type SetStatusParams = __Infer; +export type TipParams = __Infer; + diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/user_table.ts b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/user_table.ts new file mode 100644 index 00000000000..f526e35d2a2 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/module_bindings/user_table.ts @@ -0,0 +1,17 @@ +// THIS FILE IS AUTOMATICALLY GENERATED BY SPACETIMEDB. EDITS TO THIS FILE +// WILL NOT BE SAVED. MODIFY TABLES IN YOUR MODULE SOURCE CODE INSTEAD. + +/* eslint-disable */ +/* tslint:disable */ +import { + TypeBuilder as __TypeBuilder, + t as __t, + type AlgebraicTypeType as __AlgebraicTypeType, + type Infer as __Infer, +} from "spacetimedb"; + +export default __t.row({ + username: __t.string().primaryKey(), + status: __t.string(), + balance: __t.i32(), +}); diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/package-lock.json b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/package-lock.json new file mode 100644 index 00000000000..091d0fd6bd4 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/package-lock.json @@ -0,0 +1,677 @@ +{ + "name": "stack-bench-verifier-teamchat-spacetimedb", + "version": "0.0.1", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "stack-bench-verifier-teamchat-spacetimedb", + "version": "0.0.1", + "dependencies": { + "spacetimedb": "2.5.0" + }, + "devDependencies": { + "tsx": "^4.19.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.1.tgz", + "integrity": "sha512-Svl7tq8k/08+p6CXPpRjQ1fKX+1odH/BQbb48fV6fj3CWHhsoIOoY87w1oHXm0qEpkIK3ZfVgp0hed3XBXzXMQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.28.1.tgz", + "integrity": "sha512-0k2F129Xdio1TdJfzJ8sy1Q47vUD2NnwdhiAf7drUN1EBTfPf4hsFCtmMgu/6m8JSzsBrlmVjudMBQqOfG8usQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.28.1.tgz", + "integrity": "sha512-34EGEbCIAgosYz6goLcopX6Mo7NyGv9tfwEM2/7Ce2VcVRk568iSvniGWcUXIy7wEDR1wzolcxcriFVrWYcwBg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.28.1.tgz", + "integrity": "sha512-dbwY7ltSMDWsRatcRpCnES4F+im88OCUgGZjy52shC7GqHRE/cYlxNbB4Z4UpJswpcc4Qxd2oE/ufM0p61IKng==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.28.1.tgz", + "integrity": "sha512-TZbWkQY7kvTAXbXUT7uVACR5cMHsDiSz9z7ZKAX/RTq/WJEk3QyRr0wZpNhBDX+/0CtdqUIJlOiodQcta6tY3Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.28.1.tgz", + "integrity": "sha512-zfdzgK9ACBNZLI/CyHTOx81SyNbM6YXn7rxSgX97VjyiPl9W1i4Ka4fgKECEoFCKGpvBj5qArWIGgQjOwkgskQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.28.1.tgz", + "integrity": "sha512-wG2EA8ENdEI0qhkSZMjfqrdY+ziCYCPMmtZjjIwOmXFjmyzEHn+UUxk5of+SYsjtfs3VpnlC7QLzSI5hY/rOAw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.28.1.tgz", + "integrity": "sha512-i7dZ9vQgnvSCzi/rYCXNgtF/U+eKZNJBzu3eTQbRgHnM7tNSizLOkRFAl3qzVc/Op/u5YkHHa4pf/3DOYHthLQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.28.1.tgz", + "integrity": "sha512-qVXBOHQS+d5Y722GwJzJUtOLlX7km3CraOaGormF1pDtPd2C/l1SHRPgjLunLGe51Sh5YYWKMFDyV4SxgMQYTQ==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.28.1.tgz", + "integrity": "sha512-yHs+0uc8+nvEAfAfxrWQKK5peSNzBc4PegcMO0EJ2hT71uA7vB8Ihg2e77R2P7SG5uYjPbHlLLmve4LLLRCf0g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.28.1.tgz", + "integrity": "sha512-d1z4ZuP0ajrfz/FhGT4vv278rX8KnPPJx8i5+AtK7TYbx9Le9F1hyzurZpkEyjkGa9dUGhQow4C1NmeGvqxN2w==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.28.1.tgz", + "integrity": "sha512-M5sRjUVZrkm1OAPR3dlOYzNmN+loZKGVi1VUQGrwuqLcbR6qeAz+famMhjASeH3YVKvZz+zT1jlh/keC3Rj/lg==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.28.1.tgz", + "integrity": "sha512-mRObBZeHh2OxcBFPWE/FjylkRgZdYuiTR3vaTozquCGOH14iP9oN4x4Ge81CoIDYQrXmIxpFumJBu5MtZpnQJQ==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.28.1.tgz", + "integrity": "sha512-slScBsMAb3GFDcdrCgLwZtPYRoH2H/youv10QiZyRjmsP48fznoveWytSgCI/R0ZcUgpc0ZhIUEx6LHts8yrfQ==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.28.1.tgz", + "integrity": "sha512-kw0owk1o0GFETUJyW0jc0G4Yzs0BHZn0JDZ8JRT088vjJYX777BAs1fDGxAC+q831qOs2DTC96mNsG2opdfyyQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.28.1.tgz", + "integrity": "sha512-/lAIjX8aYFRByhh6L5rYtPEDRqa9de/4V/juOXcta5frjvzXO4/sqEtyytse0g3zZFuWu5cDN0MkLz2qRDD2Ag==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.28.1.tgz", + "integrity": "sha512-u/anNYF2mmVOEDwLtnQ1wOr3EZ9sTNGLWrsYGYwHWzGA3Si84IOkHXlbWTD1NB+9/1lcnweYKO54uhxZydNzfA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.28.1.tgz", + "integrity": "sha512-oks0DYbLwWMmaakTsCb+zL4E+aHRVLom9IJZOAthMQEPiQmydXHkziYEsGYRx0uNV/IjEKGAV941JzH02pflqw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.28.1.tgz", + "integrity": "sha512-aeL6lAnN89Hz43Mlh1G8ARasbuoYvSITDEx0tHh5b7jJnHcssqgjy9Yx430GDpmCa6OyrKoS0aNRjKundRizGg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.28.1.tgz", + "integrity": "sha512-MEFJe5C3R8pwXdZ5Y21oo6m7ePiS0d9pWucn99O/wvyJZChoIQKrQDxKrGeW8F5+T0okTHesAmDeiHDTIq0V/Q==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.28.1.tgz", + "integrity": "sha512-i/ZLIOafE0Z8cI/XANJAixoJL/uRAoS2xOA3rb0xN+KK0K177cMAsQYkzHtBrtMXAKuAc7HGgcWiZ/sRC1Nxgw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openharmony-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/openharmony-arm64/-/openharmony-arm64-0.28.1.tgz", + "integrity": "sha512-ge+Z7EXFNt2BO1oAMsVpiQ8EwndV9i1xXerAeTIK7AtPs3bKFXQM7nlRxDSIUIMeueR1CNXxqztLzdNeReKBJg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.28.1.tgz", + "integrity": "sha512-BEjgtECkL3vY+SaSQ6nzVfiALUeFxpawyp8Jmf5PtYhf1Ug40N1h/hxlhts+f1FvSvarEigdxS3BlSMI2PJLcQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.28.1.tgz", + "integrity": "sha512-lCv9eK/H6ZJWbE7bh2nw54CZ9M2nupBxJcTsdk/QQnWkdSjKGuxmmH8/GWrlT1eMmZfn4dGcCjRte397WqfQXA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.28.1.tgz", + "integrity": "sha512-zvb/mB2bSCoJOpoCBgYKKpX6YM6mJBlBUVUtVj41DlZJVEB6/0CKlRYxP5wWl1C1ILiCoAU5wZZ4q1P3qeS6Eg==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.28.1.tgz", + "integrity": "sha512-bm4Mowrv+GXMlpWX++EcXw/iLyd1o3+bJkC2DkWXYVvgZCqD/bSj9ctZeAMC3cIxgjRVR2Dufaiu4YPxr5gW1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/esbuild": { + "version": "0.28.1", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.28.1.tgz", + "integrity": "sha512-HrJrvZv5ayxBzPfwphOoNzkzOIIlifzk0KJrGK2c8R4+LKpMtpYLQeUdjnwjWv/LZlkH2laZk+4w78pi99D4Vw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.28.1", + "@esbuild/android-arm": "0.28.1", + "@esbuild/android-arm64": "0.28.1", + "@esbuild/android-x64": "0.28.1", + "@esbuild/darwin-arm64": "0.28.1", + "@esbuild/darwin-x64": "0.28.1", + "@esbuild/freebsd-arm64": "0.28.1", + "@esbuild/freebsd-x64": "0.28.1", + "@esbuild/linux-arm": "0.28.1", + "@esbuild/linux-arm64": "0.28.1", + "@esbuild/linux-ia32": "0.28.1", + "@esbuild/linux-loong64": "0.28.1", + "@esbuild/linux-mips64el": "0.28.1", + "@esbuild/linux-ppc64": "0.28.1", + "@esbuild/linux-riscv64": "0.28.1", + "@esbuild/linux-s390x": "0.28.1", + "@esbuild/linux-x64": "0.28.1", + "@esbuild/netbsd-arm64": "0.28.1", + "@esbuild/netbsd-x64": "0.28.1", + "@esbuild/openbsd-arm64": "0.28.1", + "@esbuild/openbsd-x64": "0.28.1", + "@esbuild/openharmony-arm64": "0.28.1", + "@esbuild/sunos-x64": "0.28.1", + "@esbuild/win32-arm64": "0.28.1", + "@esbuild/win32-ia32": "0.28.1", + "@esbuild/win32-x64": "0.28.1" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/headers-polyfill": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/headers-polyfill/-/headers-polyfill-4.0.3.tgz", + "integrity": "sha512-IScLbePpkvO846sIwOtOTDjutRMWdXdJmXdMvk6gCBHxFO8d+QKOQedyZSxFTTFYRSmlgSTDtXqqq4pcenBXLQ==", + "license": "MIT" + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/prettier": { + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.6.tgz", + "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/pure-rand": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-7.0.1.tgz", + "integrity": "sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "license": "MIT", + "engines": { + "node": ">=10" + } + }, + "node_modules/spacetimedb": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/spacetimedb/-/spacetimedb-2.5.0.tgz", + "integrity": "sha512-Np87NrFphHOvpPlsyeQQhjmswAedzr9mMW2pS1nQrYVHBJFw5OcH3RvgU4YdBvFz9MXH1owVVNvAASPv6Rvvug==", + "license": "ISC", + "dependencies": { + "base64-js": "^1.5.1", + "headers-polyfill": "^4.0.3", + "object-inspect": "^1.13.4", + "prettier": "^3.3.3", + "pure-rand": "^7.0.1", + "safe-stable-stringify": "^2.5.0", + "statuses": "^2.0.2", + "url-polyfill": "^1.1.14" + }, + "peerDependencies": { + "@angular/core": ">=17.0.0", + "@tanstack/react-query": "^5.0.0", + "react": "^18.0.0 || ^19.0.0-0 || ^19.0.0", + "solid-js": "^1.6.0", + "svelte": "^4.0.0 || ^5.0.0", + "undici": "^6.19.2", + "vue": "^3.3.0" + }, + "peerDependenciesMeta": { + "@angular/core": { + "optional": true + }, + "@tanstack/react-query": { + "optional": true + }, + "react": { + "optional": true + }, + "solid-js": { + "optional": true + }, + "svelte": { + "optional": true + }, + "undici": { + "optional": true + }, + "vue": { + "optional": true + } + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/tsx": { + "version": "4.23.5", + "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.5.tgz", + "integrity": "sha512-rw55FUaqOoI7RvlQwLbhO4nSDApnQ4/CykPuiQ/EPvtrX3WA9Ig55jIt9VvbBJbzJuj12ueRu4PMZ2SxPVbihg==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "~0.28.0" + }, + "bin": { + "tsx": "dist/cli.mjs" + }, + "engines": { + "node": ">=18.0.0" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + } + }, + "node_modules/url-polyfill": { + "version": "1.1.14", + "resolved": "https://registry.npmjs.org/url-polyfill/-/url-polyfill-1.1.14.tgz", + "integrity": "sha512-p4f3TTAG6ADVF3mwbXw7hGw+QJyw5CnNGvYh5fCuQQZIiuKUswqcznyV3pGDP9j0TSmC4UvRKm8kl1QsX1diiQ==", + "license": "MIT" + } + } +} diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/package.json b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/package.json new file mode 100644 index 00000000000..afb69757dc5 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/package.json @@ -0,0 +1,13 @@ +{ + "name": "stack-bench-verifier-teamchat-spacetimedb", + "version": "0.0.1", + "private": true, + "type": "module", + "description": "SpacetimeDB team-chat verifier: runs the shared behavioral scenario via the SpacetimeDB TS SDK. Version pinned to match the committed module_bindings (CLI/SDK 2.5.x).", + "dependencies": { + "spacetimedb": "2.5.0" + }, + "devDependencies": { + "tsx": "^4.19.0" + } +} diff --git a/tools/stack-bench/tasks/team-chat/spacetimedb/tests/test.sh b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/test.sh new file mode 100644 index 00000000000..a0990a2d9c1 --- /dev/null +++ b/tools/stack-bench/tasks/team-chat/spacetimedb/tests/test.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Thin launcher — Harbor's verifier entry point. Runs the shared team-chat +# behavioral scenario through the SpacetimeDB adapter; the harness writes +# reward.txt / reward.json / result.json under /logs/verifier. +set -euo pipefail +cd "$(dirname "$0")" +npm install --no-audit --no-fund --silent +export RESTART_CMD="${RESTART_CMD:-/opt/stack-bench/backendctl restart}" +exec npx tsx harness/src/runTeamChat.ts "$(pwd)/adapter.ts" diff --git a/tools/stack-bench/xtask/Cargo.toml b/tools/stack-bench/xtask/Cargo.toml new file mode 100644 index 00000000000..535251bde1c --- /dev/null +++ b/tools/stack-bench/xtask/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "xtask-stack-bench" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +default-run = "stack_bench" + +[lints] +workspace = true + +[[bin]] +name = "stack_bench" +path = "src/main.rs" + +[dependencies] +anyhow.workspace = true +clap = { workspace = true, features = ["derive"] } +serde_json.workspace = true diff --git a/tools/stack-bench/xtask/src/main.rs b/tools/stack-bench/xtask/src/main.rs new file mode 100644 index 00000000000..54801cc8e0b --- /dev/null +++ b/tools/stack-bench/xtask/src/main.rs @@ -0,0 +1,305 @@ +//! stack-bench runner. +//! +//! Drives the cross-backend agentic benchmark via Harbor, so the whole workflow +//! is `cargo stack-bench …` instead of shell glue. The only shell that remains is +//! the per-task `solve.sh`/`test.sh`, which Harbor *requires* as its oracle/verifier +//! entry points (on Linux it discovers only `*.sh`); those are thin launchers. +//! +//! cargo stack-bench build # inject the shared grader into tasks +//! cargo stack-bench list # list backends +//! cargo stack-bench oracle spacetimedb # one backend, oracle (expect reward 1.0) +//! cargo stack-bench agent convex --model anthropic/claude-opus-4-6 +//! cargo stack-bench all --model anthropic/claude-opus-4-6 # every backend + compare +//! +//! Extra args after `--` are forwarded to `harbor run`, e.g. +//! cargo stack-bench oracle convex -- -n 1 + +use anyhow::{bail, Context, Result}; +use clap::{Parser, Subcommand}; +use serde_json::Value; +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::SystemTime; + +#[derive(Parser)] +#[command(name = "stack-bench", about = "Run the stack-bench cross-backend agentic benchmark (Harbor)")] +struct Cli { + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// Inject the shared grader (_shared/harness) into each task's tests/harness. + Build, + /// List available backends (task variants under tasks/realtime-chat/). + List, + /// Run one backend with the oracle solution (sanity check; expect reward 1.0). + Oracle { + /// Backend, e.g. `spacetimedb` or `convex`. + backend: String, + /// Task family under tasks/ (e.g. `team-chat`, `realtime-chat`). + #[arg(long, default_value = "team-chat")] + task: String, + /// Args forwarded verbatim to `harbor run` (after `--`). + #[arg(last = true)] + extra: Vec, + }, + /// Run one backend with a real agent. + Agent { + /// Backend, e.g. `spacetimedb` or `convex`. + backend: String, + /// Task family under tasks/ (e.g. `team-chat`, `realtime-chat`). + #[arg(long, default_value = "team-chat")] + task: String, + /// Harbor agent name. + #[arg(long, default_value = "claude-code")] + agent: String, + /// Model, e.g. `anthropic/claude-opus-4-6` or `openrouter/anthropic/claude-opus-4-6`. + #[arg(long)] + model: Option, + /// Args forwarded verbatim to `harbor run` (after `--`). + #[arg(last = true)] + extra: Vec, + }, + /// Run EVERY backend with the same agent+model and print a comparison table. + /// This is the product view: rank backends for a fixed agent. + All { + /// Harbor agent name (default: oracle — the harness self-test). + #[arg(long, default_value = "oracle")] + agent: String, + /// Model, e.g. `anthropic/claude-opus-4-6` (ignored by the oracle agent). + #[arg(long)] + model: Option, + /// Base directory for per-backend job results (default: /jobs). + #[arg(long)] + jobs_dir: Option, + /// Args forwarded verbatim to each `harbor run` (after `--`). + #[arg(last = true)] + extra: Vec, + }, +} + +/// Benchmark root = this crate's parent dir (crate lives at /xtask). +fn root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask has a parent dir") + .to_path_buf() +} + +fn tasks_dir() -> PathBuf { + root().join("tasks") +} + +/// All (task, backend) pairs: every tasks/// with a task.toml. +fn variants() -> Result> { + let mut out = Vec::new(); + for task in fs::read_dir(tasks_dir()).context("read tasks/")? { + let task = task?; + if !task.path().is_dir() { + continue; + } + let task_name = task.file_name().to_string_lossy().to_string(); + for backend in fs::read_dir(task.path())? { + let backend = backend?; + if backend.path().join("task.toml").exists() { + out.push((task_name.clone(), backend.file_name().to_string_lossy().to_string())); + } + } + } + out.sort(); + Ok(out) +} + +fn task_path(task: &str, backend: &str) -> Result { + let p = tasks_dir().join(task).join(backend); + if !p.join("task.toml").exists() { + let avail = variants()? + .iter() + .map(|(t, b)| format!("{t}/{b}")) + .collect::>() + .join(", "); + bail!("unknown task/backend '{task}/{backend}'. available: {avail}"); + } + Ok(p) +} + +/// Recursively copy `src` into `dst`, skipping build/vendor dirs. +fn copy_dir(src: &Path, dst: &Path) -> Result<()> { + fs::create_dir_all(dst).with_context(|| format!("create {}", dst.display()))?; + for entry in fs::read_dir(src).with_context(|| format!("read {}", src.display()))? { + let entry = entry?; + let name = entry.file_name(); + if matches!(name.to_str(), Some("node_modules" | "dist" | ".git")) { + continue; + } + let from = entry.path(); + let to = dst.join(&name); + if from.is_dir() { + copy_dir(&from, &to)?; + } else { + fs::copy(&from, &to).with_context(|| format!("copy {} -> {}", from.display(), to.display()))?; + } + } + Ok(()) +} + +/// Inject _shared/harness into every task's tests/harness (replaces build-task.sh). +/// Harbor tasks must be self-contained, so we keep one source of truth and copy it in. +fn inject_harness() -> Result<()> { + let harness = root().join("_shared/harness"); + if !harness.is_dir() { + bail!("missing shared harness at {}", harness.display()); + } + let mut n = 0; + for (task, backend) in variants()? { + let tests = tasks_dir().join(&task).join(&backend).join("tests"); + if !tests.is_dir() { + continue; + } + let dest = tests.join("harness"); + if dest.exists() { + fs::remove_dir_all(&dest).with_context(|| format!("rm {}", dest.display()))?; + } + copy_dir(&harness, &dest)?; + println!("injected harness -> tasks/{task}/{backend}/tests/harness"); + n += 1; + } + println!("done: injected harness into {n} task(s)"); + Ok(()) +} + +fn harbor_run( + task: &Path, + agent: &str, + model: Option<&str>, + out_dir: Option<&Path>, + extra: &[String], +) -> Result<()> { + let mut cmd = Command::new("harbor"); + cmd.arg("run").arg("-p").arg(task).arg("-a").arg(agent).arg("-y"); + if let Some(m) = model { + cmd.arg("-m").arg(m); + } + if let Some(o) = out_dir { + cmd.arg("-o").arg(o); + } + cmd.args(extra); + eprintln!( + "+ harbor run -p {} -a {agent}{}", + task.display(), + model.map(|m| format!(" -m {m}")).unwrap_or_default() + ); + let status = cmd + .status() + .context("failed to spawn `harbor` — is it installed and on PATH? (`uv tool install harbor`)")?; + if !status.success() { + bail!("harbor exited with {status}"); + } + Ok(()) +} + +/// One backend's run summary, parsed from Harbor's result.json. +struct Summary { + reward: Option, + errored: i64, + in_tokens: Option, + out_tokens: Option, +} + +/// Find the most-recently-written result.json under `dir` and parse the reward + token stats. +fn read_latest_summary(dir: &Path) -> Result { + let mut newest: Option<(SystemTime, PathBuf)> = None; + let mut stack = vec![dir.to_path_buf()]; + while let Some(d) = stack.pop() { + let Ok(entries) = fs::read_dir(&d) else { continue }; + for e in entries.flatten() { + let p = e.path(); + if p.is_dir() { + stack.push(p); + } else if p.file_name().and_then(|n| n.to_str()) == Some("result.json") { + let mtime = e.metadata().and_then(|m| m.modified()).unwrap_or(SystemTime::UNIX_EPOCH); + if newest.as_ref().is_none_or(|(t, _)| mtime >= *t) { + newest = Some((mtime, p)); + } + } + } + } + let (_, path) = newest.with_context(|| format!("no result.json found under {}", dir.display()))?; + let v: Value = serde_json::from_str(&fs::read_to_string(&path)?) + .with_context(|| format!("parse {}", path.display()))?; + // reward = mean of the (single) eval in stats.evals + let reward = v["stats"]["evals"] + .as_object() + .and_then(|m| m.values().next()) + .and_then(|eval| eval["metrics"].get(0)) + .and_then(|m| m["mean"].as_f64()); + Ok(Summary { + reward, + errored: v["stats"]["n_errored_trials"].as_i64().unwrap_or(0), + in_tokens: v["stats"]["n_input_tokens"].as_i64(), + out_tokens: v["stats"]["n_output_tokens"].as_i64(), + }) +} + +fn run_all(agent: &str, model: Option<&str>, jobs_dir: Option, extra: &[String]) -> Result<()> { + inject_harness()?; + let base = jobs_dir.unwrap_or_else(|| root().join("jobs")); + let mut rows: Vec<(String, Result)> = Vec::new(); + for (task, backend) in variants()? { + let path = task_path(&task, &backend)?; + let out = base.join(&task).join(&backend); + let summary = harbor_run(&path, agent, model, Some(&out), extra).and_then(|()| read_latest_summary(&out)); + rows.push((format!("{task}/{backend}"), summary)); + } + print_comparison(agent, model, &rows); + Ok(()) +} + +fn print_comparison(agent: &str, model: Option<&str>, rows: &[(String, Result)]) { + let tok = |t: Option| t.map(|n| n.to_string()).unwrap_or_else(|| "-".to_string()); + println!("\n=== stack-bench comparison — agent={agent} model={} ===", model.unwrap_or("-")); + println!("{:<24} {:>8} {:>7} {:>10} {:>10}", "backend", "reward", "errors", "in_tok", "out_tok"); + println!("{:-<24} {:->8} {:->7} {:->10} {:->10}", "", "", "", "", ""); + for (backend, res) in rows { + match res { + Ok(s) => println!( + "{:<24} {:>8} {:>7} {:>10} {:>10}", + backend, + s.reward.map(|r| format!("{r:.3}")).unwrap_or_else(|| "?".to_string()), + s.errored, + tok(s.in_tokens), + tok(s.out_tokens), + ), + Err(e) => println!("{backend:<24} {:>8} ({e})", "ERR"), + } + } + println!(); +} + +fn main() -> Result<()> { + match Cli::parse().cmd { + Cmd::Build => inject_harness()?, + Cmd::List => { + for (t, b) in variants()? { + println!("{t}/{b}"); + } + } + Cmd::Oracle { backend, task, extra } => { + let path = task_path(&task, &backend)?; + inject_harness()?; + harbor_run(&path, "oracle", None, None, &extra)?; + } + Cmd::Agent { backend, task, agent, model, extra } => { + let path = task_path(&task, &backend)?; + inject_harness()?; + harbor_run(&path, &agent, model.as_deref(), None, &extra)?; + } + Cmd::All { agent, model, jobs_dir, extra } => { + run_all(&agent, model.as_deref(), jobs_dir, &extra)?; + } + } + Ok(()) +}