diff --git a/.github/workflows/sdk-matrix.yml b/.github/workflows/sdk-matrix.yml new file mode 100644 index 00000000..6d33239e --- /dev/null +++ b/.github/workflows/sdk-matrix.yml @@ -0,0 +1,339 @@ +name: SDK matrix + +# Runs one conformance selection (a scenario list, a suite, or a requirement +# set) against every SDK in KNOWN_SDKS (src/sdk-runner/known-sdks.ts) and +# renders an SDK x check table, so the cross-SDK impact of a harness change can +# be checked by any maintainer in one command instead of an ad-hoc set of local +# toolchains. Example: post the table for a client-side check change onto its PR: +# +# gh workflow run sdk-matrix.yml -R modelcontextprotocol/conformance \ +# -f sdks=all -f mode=client \ +# -f scenario=auth/metadata-var2,auth/metadata-default \ +# -f pr=488 -f pr_comment=true +# +# `pr` selects what to test (the PR's merge ref, or its merge commit once the +# PR has merged) and where the optional sticky comment goes; `ref` overrides +# what to test. With neither, the dispatched branch is tested. The weekly +# schedule runs both modes with each SDK's default suites and only writes the +# step summary. +# +# Security model (the same split as traceability.yml): +# - `plan` and `run` execute code from the ref under test, and `run` also +# builds and runs third-party SDK code. Both get a read-only token scope, no +# persisted git credentials and no secrets; the SDK build/run step never has +# a token in its environment. +# - `report` runs no SDK code and no code from the ref under test: it checks +# out the dispatching branch, merges the uploaded JSON with its own copy of +# the script, and writes the step summary. Still read-only. +# - `comment` is the only job with a write permission (pull-requests). It checks +# nothing out and runs no repository or SDK code; it posts the rendered table, +# read from the artifact, as data. +# - The workflow is dispatch/schedule only, so it always runs as defined on a +# branch of this repository. Testing a fork PR via `pr=` runs the fork's +# harness code only inside the unprivileged jobs, and `plan` flags it. Those +# jobs use no actions caches (a dispatch runs in the default branch's cache +# scope, so nothing an untrusted build could influence is saved or restored). + +on: + workflow_dispatch: + inputs: + sdks: + description: 'SDKs to run: "all" or a comma-separated list of KNOWN_SDKS names, each optionally name@ref (e.g. go-sdk@v1.2.0,rust-sdk)' + default: 'all' + mode: + description: 'Side to test' + type: choice + options: [client, server, both] + default: client + scenario: + description: 'Scenario name(s), comma-separated (e.g. auth/metadata-default). Leave scenario/suite/requirements all empty for the sdk command defaults.' + suite: + description: 'Suite to run instead of scenarios (e.g. auth, all, active)' + requirements: + description: 'Requirement set to run instead (e.g. 2026-07-28)' + ref: + description: 'Conformance branch, tag or sha to test. Default: the PR merge ref when `pr` is set, else the branch this was dispatched on.' + pr: + description: 'Conformance PR number. Tests refs/pull//merge (or the merge commit if already merged) and is where pr_comment posts.' + pr_comment: + description: 'Upsert a sticky comment with the table on that PR' + type: boolean + default: false + schedule: + - cron: '0 7 * * 1' # Weekly, Monday 07:00 UTC (after the traceability refresh). + +permissions: + contents: read + +concurrency: + group: sdk-matrix-${{ inputs.pr || inputs.ref || github.ref }}-${{ inputs.mode || 'both' }}-${{ inputs.scenario || inputs.suite || inputs.requirements || 'default' }} + cancel-in-progress: true + +env: + SDKS: ${{ inputs.sdks || 'all' }} + MODE: ${{ inputs.mode || 'both' }} + SCENARIO: ${{ inputs.scenario }} + SUITE: ${{ inputs.suite }} + REQUIREMENTS: ${{ inputs.requirements }} + PR: ${{ inputs.pr }} + REF_INPUT: ${{ inputs.ref }} + +jobs: + plan: + runs-on: ubuntu-latest + permissions: + contents: read + outputs: + ref: ${{ steps.ref.outputs.ref }} + label: ${{ steps.ref.outputs.label }} + sha: ${{ steps.sha.outputs.sha }} + matrix: ${{ steps.sdks.outputs.matrix }} + steps: + - name: Resolve the conformance ref under test + id: ref + env: + GH_TOKEN: ${{ github.token }} # read-only; only used to look up a merged PR's merge commit + run: | + set -euo pipefail + if [ -n "$PR" ] && ! [[ "$PR" =~ ^[0-9]+$ ]]; then + echo "::error::pr must be a number (got '$PR')"; exit 1 + fi + if [ -n "$REF_INPUT" ]; then + ref="$REF_INPUT"; label="$REF_INPUT" + [ -n "$PR" ] && label="$REF_INPUT (for PR #$PR)" + elif [ -n "$PR" ]; then + label="PR #$PR" + if git ls-remote --exit-code "https://github.com/$GITHUB_REPOSITORY.git" "refs/pull/$PR/merge" >/dev/null 2>&1; then + ref="refs/pull/$PR/merge" + else + # Merged (or conflicting) PRs have no merge ref; test the merge commit. + ref="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq '.merge_commit_sha // empty' || true)" + if [ -z "$ref" ]; then ref="refs/pull/$PR/head"; fi + label="PR #$PR (merged)" + fi + else + ref="$GITHUB_SHA"; label="${GITHUB_REF_NAME}" + fi + if [ -n "$PR" ]; then + head_repo="$(gh api "repos/$GITHUB_REPOSITORY/pulls/$PR" --jq '.head.repo.full_name // empty' || true)" + if [ -n "$head_repo" ] && [ "$head_repo" != "$GITHUB_REPOSITORY" ]; then + echo "::warning::PR #$PR comes from a fork ($head_repo). Its harness code runs unprivileged in the run jobs; dispatching it is the same trust decision as approving CI for that PR." + fi + fi + echo "ref=$ref" >> "$GITHUB_OUTPUT" + echo "label=$label" >> "$GITHUB_OUTPUT" + echo "Testing conformance at: $ref ($label)" + + # The dispatching branch supplies the orchestration script; the ref under + # test supplies KNOWN_SDKS (it may predate the script). + - uses: actions/checkout@v6 + with: + path: tooling + persist-credentials: false + - uses: actions/checkout@v6 + with: + ref: ${{ steps.ref.outputs.ref }} + path: under-test + persist-credentials: false + - id: sha + run: echo "sha=$(git -C under-test rev-parse --short=12 HEAD)" >> "$GITHUB_OUTPUT" + - uses: actions/setup-node@v6 + with: + node-version: 22 + - name: Expand the SDK list into a job matrix + id: sdks + run: | + set -euo pipefail + if [ "$SDKS" = "all" ]; then + list="$(node tooling/scripts/sdk-matrix.mjs --harness-dir under-test --list-sdks --json)" + else + list="$(jq -cn --arg s "$SDKS" '$s | split(",") | map(gsub("^\\s+|\\s+$"; "")) | map(select(length > 0))')" + fi + matrix="$(jq -cn --argjson l "$list" '$l | map({spec: ., id: gsub("[^A-Za-z0-9._-]"; "_")})')" + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" + echo "SDK legs: $matrix" + + run: + needs: plan + name: run (${{ matrix.spec }}) + runs-on: ubuntu-latest + timeout-minutes: 60 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.matrix) }} + env: + SPEC: ${{ matrix.spec }} + steps: + - uses: actions/checkout@v6 + with: + path: tooling + persist-credentials: false + - uses: actions/checkout@v6 + with: + ref: ${{ needs.plan.outputs.ref }} + path: under-test + persist-credentials: false # no git token on disk while SDK code runs + + # Toolchains, keyed off the SDK name. Node is always needed (the harness). + # No actions/cache use anywhere in this job (setup-* caches disabled): + # it executes code from the ref under test, which for a fork PR is + # untrusted, and a dispatch runs in the default branch's cache scope. + - uses: actions/setup-node@v6 + with: + node-version: 22 + - name: Enable corepack (pnpm for typescript-sdk) + if: contains(matrix.spec, 'typescript-sdk') + run: corepack enable + - uses: astral-sh/setup-uv@v7 + if: contains(matrix.spec, 'python-sdk') + with: + enable-cache: false + - uses: actions/setup-go@v6 + if: contains(matrix.spec, 'go-sdk') + with: + go-version: stable + cache: false + - uses: dtolnay/rust-toolchain@stable + if: contains(matrix.spec, 'rust-sdk') + - name: Resolve the .NET SDK version csharp-sdk pins (global.json) + if: contains(matrix.spec, 'csharp-sdk') + id: dotnet + run: | + set -euo pipefail + name="${SPEC%@*}"; ref="main" + case "$SPEC" in *@*) ref="${SPEC##*@}";; esac + case "$name" in */*) repo="$name";; *) repo="modelcontextprotocol/$name";; esac + v="$(curl -fsSL "https://raw.githubusercontent.com/$repo/$ref/global.json" | jq -r '.sdk.version // empty' | sed -E 's/^([0-9]+\.[0-9]+)\..*/\1.x/' || true)" + echo "version=${v:-10.0.x}" >> "$GITHUB_OUTPUT" + echo "dotnet-version: ${v:-10.0.x} (from $repo@$ref global.json)" + - uses: actions/setup-dotnet@v5 + if: contains(matrix.spec, 'csharp-sdk') + with: + dotnet-version: ${{ steps.dotnet.outputs.version }} + - uses: ruby/setup-ruby@v1 + if: contains(matrix.spec, 'ruby-sdk') + with: + ruby-version: '4.0' # what ruby-sdk's own conformance CI runs + - uses: actions/setup-java@v5 + if: contains(matrix.spec, 'java-sdk') || contains(matrix.spec, 'kotlin-sdk') + with: + distribution: temurin + java-version: 21 + + - name: Build the harness under test + working-directory: under-test + run: npm ci && npm run build + + - name: Run the matrix leg for this SDK + # No token here: this step clones, builds and runs third-party SDK code. + # --strict-errors turns this leg red only when the SDK could not be + # built or run at all; check failures are results, reported in the table. + env: + SDK_MATRIX_HARNESS_REF: ${{ needs.plan.outputs.label }} + SDK_MATRIX_HARNESS_SHA: ${{ needs.plan.outputs.sha }} + run: | + node tooling/scripts/sdk-matrix.mjs \ + --harness-dir under-test \ + --sdks "$SPEC" \ + --mode "$MODE" \ + --scenario "$SCENARIO" --suite "$SUITE" --requirements "$REQUIREMENTS" \ + --cache-dir "$RUNNER_TEMP/sdk-under-test" \ + --title "SDK matrix: conformance ${SDK_MATRIX_HARNESS_REF} @ ${SDK_MATRIX_HARNESS_SHA}" \ + --strict-errors \ + -o results + + - uses: actions/upload-artifact@v4 + if: ${{ !cancelled() }} + with: + name: sdk-matrix-${{ matrix.id }} + path: results + retention-days: 14 + if-no-files-found: error + + report: + needs: [plan, run] + if: ${{ !cancelled() && needs.plan.result == 'success' }} + runs-on: ubuntu-latest + permissions: + contents: read + steps: + # The dispatching branch's script only; nothing from the ref under test + # and no SDK code runs in this job. + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-node@v6 + with: + node-version: 22 + - uses: actions/download-artifact@v4 + with: + pattern: sdk-matrix-* + path: artifacts + - name: Merge legs and write the step summary + env: + LABEL: ${{ needs.plan.outputs.label }} + SHA: ${{ needs.plan.outputs.sha }} + run: | + set -euo pipefail + node scripts/sdk-matrix.mjs --merge artifacts -o merged \ + --title "SDK matrix: conformance ${LABEL} @ ${SHA}" > /dev/null + cat merged/matrix.md >> "$GITHUB_STEP_SUMMARY" + - uses: actions/upload-artifact@v4 + with: + name: sdk-matrix + path: merged + retention-days: 30 + + comment: + needs: [plan, report] + if: ${{ !cancelled() && needs.report.result == 'success' && inputs.pr != '' && inputs.pr_comment == true }} + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - uses: actions/download-artifact@v4 + with: + name: sdk-matrix + path: merged + - name: Upsert the sticky PR comment + uses: actions/github-script@v8 + env: + PR_NUMBER: ${{ inputs.pr }} + with: + script: | + const fs = require('fs'); + const marker = ''; + const issue_number = Number(process.env.PR_NUMBER); + if (!Number.isInteger(issue_number) || issue_number <= 0) { + core.setFailed(`Invalid PR number: ${process.env.PR_NUMBER}`); + return; + } + // The table is data produced from SDK output; it is read from the + // artifact and never interpolated into this script. + let table = fs.readFileSync('merged/matrix.md', 'utf8'); + const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; + const footer = `\n\nPosted by the [sdk-matrix workflow run](${runUrl}); full logs and matrix.json are in that run's artifacts. Re-running the workflow with the same PR number updates this comment.\n`; + const max = 60000; + if (table.length > max) { + table = table.slice(0, table.lastIndexOf('\n', max)) + + '\n\n(Truncated; the full table is in the run summary and the sdk-matrix artifact.)'; + } + const body = `${marker}\n${table}${footer}`; + const comments = await github.paginate(github.rest.issues.listComments, { + ...context.repo, + issue_number, + per_page: 100 + }); + const existing = comments.find( + (c) => c.user && c.user.type === 'Bot' && typeof c.body === 'string' && c.body.startsWith(marker) + ); + if (existing) { + await github.rest.issues.updateComment({ ...context.repo, comment_id: existing.id, body }); + core.info(`Updated comment ${existing.html_url}`); + } else { + const { data } = await github.rest.issues.createComment({ ...context.repo, issue_number, body }); + core.info(`Created comment ${data.html_url}`); + } diff --git a/.gitignore b/.gitignore index 2b6bcd5b..fb3d4a51 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ dist/ .claude/settings.local.json .sdk-under-test/ .sync-schema-tmp/ +sdk-matrix-results/ diff --git a/AGENTS.md b/AGENTS.md index ca88ff87..dca410de 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -122,5 +122,6 @@ Use the existing CLI runner (`npx @modelcontextprotocol/conformance client|serve - `npm run build` passes - `npm test` passes - For non-trivial scenario changes, run against at least one real SDK (typescript-sdk or python-sdk) to see actual output. For changes to shared infrastructure (runner, tier-check), test against go-sdk or csharp-sdk too. +- If the change adds a check to an existing scenario, or changes the severity of an existing check, attach the cross-SDK table from the SDK matrix (README: "Running a Scenario Across All SDKs"; `gh workflow run sdk-matrix.yml ... -f pr= -f pr_comment=true` posts it on the PR for you). Reviewers want to see which SDKs go red before it merges, not after. - Scenario is registered in the right suite in `src/scenarios/index.ts` - If you changed a `sep-*.yaml` or scenario check IDs, `src/seps/traceability.json` will drift; the traceability workflow refreshes it via PR (or regenerate locally with `--results` from a suite run) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 920b1750..d6130160 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -68,4 +68,5 @@ See the [README](./README.md) for full CLI options and the [SDK Integration Guid - Register your scenario in the right suite in `src/scenarios/index.ts` - Run against at least one real SDK (see above) before opening the PR — we'll ask what the output looked like +- If you add a check to an existing scenario or change a check's severity, include the SDK matrix table (see "Running a Scenario Across All SDKs" in the README) so the impact on each SDK is visible in review - Keep PRs focused; one feature or scenario group at a time diff --git a/README.md b/README.md index d5c4cb3f..34af2a9a 100644 --- a/README.md +++ b/README.md @@ -394,6 +394,52 @@ To add a new SDK to the matrix, add an entry to `KNOWN_SDKS`. Clones are cached under `.sdk-under-test/` and reused (fetched) on subsequent runs. +## Running a Scenario Across All SDKs + +When a change adds a check, or changes the severity of a check on an existing scenario, the question reviewers ask is "what does this do to each SDK?". The SDK matrix answers that in one command: it runs one selection (a scenario list, a suite, or a requirement set) through `conformance sdk` for every entry in `KNOWN_SDKS` and renders an SDK x check table (`matrix.md`) plus the raw results (`matrix.json`, per-SDK logs and `checks.json` files). One SDK failing to build never stops the others; its column says why. The report leads with the question that matters for review: **Regressions**, meaning failing checks that the SDK's own expected-failures baseline does not already excuse (what would turn that SDK's CI red), plus baseline entries that now pass (stale) and SDKs that could not be run. Baselined failures are still shown, marked separately. + +There are three ways to run it, in order of preference. + +**1. GitHub Actions (no local toolchains needed).** The `sdk-matrix.yml` workflow fans out one job per SDK on hosted runners with the right toolchain, merges the results into the run summary, and can post the table as a sticky comment on a conformance PR. Any maintainer can trigger it: + +```bash +# Post the cross-SDK table for a client check onto its PR (works before or after merge) +gh workflow run sdk-matrix.yml -R modelcontextprotocol/conformance \ + -f sdks=all -f mode=client \ + -f scenario=auth/metadata-var2,auth/metadata-default \ + -f pr=488 -f pr_comment=true + +# A server scenario against two SDKs at specific refs, testing a branch of this repo +gh workflow run sdk-matrix.yml -R modelcontextprotocol/conformance \ + -f sdks=go-sdk@v1.3.0,rust-sdk -f mode=server -f scenario=tools-list -f ref=my-branch + +# Then open the run summary +gh run list -R modelcontextprotocol/conformance --workflow sdk-matrix.yml -L 1 +``` + +Inputs: `sdks` (`all` or a comma-separated list, each optionally `name@ref`), `mode` (`client`, `server`, `both`), one of `scenario` / `suite` / `requirements`, `ref` (branch, tag or sha of this repo to test), `pr` (PR number: tests its merge ref, or its merge commit once merged) and `pr_comment`. It also runs weekly against `main` with each SDK's default suites. SDK code runs in jobs that hold no token; the job that comments never runs SDK code. + +**2. Docker (local, all toolchains in one image).** `scripts/sdk-matrix-docker.sh` builds `docker/sdk-matrix` on first use (Node, uv, Go, rustup, .NET, Ruby, JDK) and runs the matrix against your checkout, with SDK clones, builds and package caches kept in a named volume so reruns take seconds: + +```bash +scripts/sdk-matrix-docker.sh --mode client --scenario auth/metadata-default +scripts/sdk-matrix-docker.sh --sdks rust-sdk,csharp-sdk,ruby-sdk --mode server --scenario tools-list +scripts/sdk-matrix-docker.sh --ref 488 --mode client --suite auth # test a PR of this repo instead of the checkout +``` + +The image uses each ecosystem's public registry. If your environment requires a mirror, forward exactly the configuration you need, for example `--npmrc ~/.npmrc --cargo-config ~/.cargo/config.toml --env GOPROXY --env UV_INDEX_URL` (see the script header for the full list). The wrapper never mounts SSH keys, git credentials or tokens, and the container only has outbound network access. + +**3. Plain local script (uses whatever toolchains are on your PATH).** + +```bash +npm run sdk-matrix -- --mode client --scenario auth/metadata-default # all SDKs +npm run sdk-matrix -- --sdks typescript-sdk,python-sdk --mode both --suite core # a subset +npm run sdk-matrix -- --merge dir1,dir2 -o combined # re-render saved results +node scripts/sdk-matrix.mjs --help +``` + +SDKs whose toolchain is missing show up as `build failed: ` rather than aborting the run. Results go to `sdk-matrix-results/` by default and the markdown table is printed to stdout, ready to paste into a PR. + ## SDK Tier Assessment The `tier-check` subcommand evaluates an MCP SDK repository against [SEP-1730](https://github.com/modelcontextprotocol/modelcontextprotocol/issues/1730) (the SDK Tiering System). There are two ways to run it, and they answer different questions. diff --git a/docker/sdk-matrix/Dockerfile b/docker/sdk-matrix/Dockerfile new file mode 100644 index 00000000..ca12e8d9 --- /dev/null +++ b/docker/sdk-matrix/Dockerfile @@ -0,0 +1,103 @@ +# Toolchain image for running the conformance SDK matrix locally with the same +# coverage as the sdk-matrix GitHub Actions workflow: Node (harness + +# typescript-sdk), uv/Python (python-sdk), Go (go-sdk), rustup (rust-sdk), +# .NET SDK (csharp-sdk), Ruby + Bundler (ruby-sdk) and a JDK (java/kotlin). +# +# Everything is installed from each toolchain's default public source. Package +# registries used at *run* time (npm, PyPI, crates.io, NuGet, RubyGems, the Go +# module proxy) are also the public defaults; scripts/sdk-matrix-docker.sh can +# forward a mirror configuration into the container for environments that +# need one. Nothing environment-specific is baked into the image. +# +# Build: docker build -t conformance-sdk-matrix --build-arg UID=$(id -u) --build-arg GID=$(id -g) docker/sdk-matrix +# Run: scripts/sdk-matrix-docker.sh --mode client --scenario initialize +FROM ubuntu:24.04 + +ARG NODE_MAJOR=22 +# Empty means "latest stable" as published by go.dev at build time. +ARG GO_VERSION="" +# csharp-sdk pins its SDK band in global.json (rollForward: minor); 10.0 today. +ARG DOTNET_CHANNEL=10.0 +ARG RUST_TOOLCHAIN=stable +ARG UID=1000 +ARG GID=1000 + +ENV DEBIAN_FRONTEND=noninteractive \ + LANG=C.UTF-8 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates curl git gnupg jq unzip xz-utils tzdata procps \ + build-essential pkg-config libssl-dev zlib1g-dev libyaml-dev libffi-dev \ + libicu74 \ + python3 python3-venv python3-dev \ + ruby-full ruby-bundler \ + openjdk-21-jdk-headless \ + && rm -rf /var/lib/apt/lists/* + +# Node.js (official tarball) + corepack, which provides the pnpm that +# typescript-sdk's packageManager field pins. +RUN set -eux; \ + arch="$(dpkg --print-architecture)"; case "$arch" in amd64) node_arch=x64;; arm64) node_arch=arm64;; *) echo "unsupported arch $arch"; exit 1;; esac; \ + base="https://nodejs.org/dist/latest-v${NODE_MAJOR}.x"; \ + file="$(curl -fsSL "$base/SHASUMS256.txt" | awk -v a="linux-${node_arch}.tar.xz" '$2 ~ a {print $2; exit}')"; \ + curl -fsSL "$base/$file" -o /tmp/node.tar.xz; \ + tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner; \ + rm /tmp/node.tar.xz; \ + corepack enable; \ + node --version; npm --version + +# uv (python-sdk); uv fetches a matching CPython on demand. +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/ + +# Go (go-sdk) +RUN set -eux; \ + arch="$(dpkg --print-architecture)"; \ + v="${GO_VERSION:-$(curl -fsSL 'https://go.dev/VERSION?m=text' | head -n1)}"; \ + case "$v" in go*) ;; *) v="go$v";; esac; \ + curl -fsSL "https://go.dev/dl/${v}.linux-${arch}.tar.gz" | tar -xz -C /usr/local; \ + /usr/local/go/bin/go version +ENV PATH=/usr/local/go/bin:$PATH + +# Rust (rust-sdk). rust-sdk pins its own channel in rust-toolchain.toml, which +# rustup installs on first use; the entrypoint moves RUSTUP_HOME onto the cache +# volume so that download happens once. +ENV RUSTUP_HOME=/opt/rust/rustup \ + CARGO_HOME=/opt/rust/cargo +RUN set -eux; \ + curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain "${RUST_TOOLCHAIN}" --no-modify-path; \ + chmod -R a+rwX /opt/rust; \ + /opt/rust/cargo/bin/cargo --version +ENV PATH=/opt/rust/cargo/bin:$PATH + +# .NET SDK (csharp-sdk) +RUN set -eux; \ + curl -fsSL https://dot.net/v1/dotnet-install.sh -o /tmp/dotnet-install.sh; \ + bash /tmp/dotnet-install.sh --channel "${DOTNET_CHANNEL}" --install-dir /opt/dotnet; \ + rm /tmp/dotnet-install.sh; \ + ln -s /opt/dotnet/dotnet /usr/local/bin/dotnet; \ + dotnet --version +ENV DOTNET_ROOT=/opt/dotnet \ + DOTNET_CLI_TELEMETRY_OPTOUT=1 \ + DOTNET_NOLOGO=1 \ + DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 \ + DOTNET_GENERATE_ASPNET_CERTIFICATE=false + +# Non-root user matching the host uid/gid so files written to the bind-mounted +# checkout stay owned by the invoking user. +RUN set -eux; \ + if id ubuntu >/dev/null 2>&1; then userdel -r ubuntu || true; fi; \ + if ! getent group "${GID}" >/dev/null; then groupadd -g "${GID}" runner; fi; \ + useradd -m -u "${UID}" -g "${GID}" -s /bin/bash runner; \ + mkdir -p /work /cache; chown "${UID}:${GID}" /work /cache + +COPY entrypoint.sh /usr/local/bin/sdk-matrix-entrypoint +RUN chmod 0755 /usr/local/bin/sdk-matrix-entrypoint + +USER runner +WORKDIR /work +VOLUME ["/cache"] +ENV SDK_MATRIX_IN_DOCKER=1 \ + COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \ + CI=1 +ENTRYPOINT ["/usr/local/bin/sdk-matrix-entrypoint"] diff --git a/docker/sdk-matrix/entrypoint.sh b/docker/sdk-matrix/entrypoint.sh new file mode 100755 index 00000000..1e365d1c --- /dev/null +++ b/docker/sdk-matrix/entrypoint.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Entrypoint for the sdk-matrix image. Expects the conformance checkout at +# /work (bind mount) and a persistent volume at /cache, then runs +# scripts/sdk-matrix.mjs with the given arguments. +set -euo pipefail + +WORK=/work +CACHE=/cache + +if [ ! -f "$WORK/scripts/sdk-matrix.mjs" ]; then + echo "sdk-matrix: expected a conformance checkout mounted at $WORK (scripts/sdk-matrix.mjs not found)" >&2 + exit 2 +fi +if ! [ -w "$CACHE" ]; then + echo "sdk-matrix: $CACHE is not writable by uid $(id -u); rebuild the image with --build-arg UID=$(id -u) or use a fresh volume" >&2 + exit 2 +fi + +# Everything a toolchain caches or installs at run time lives on the volume so +# reruns are fast: SDK clones/builds, cargo registry + pinned toolchains, Go +# module/build cache, uv's Pythons and wheels, pnpm store, NuGet packages, gems. +export HOME="$CACHE/home" +mkdir -p "$HOME" +export CARGO_HOME="$HOME/.cargo" +export RUSTUP_HOME="$CACHE/rustup" +if [ ! -d "$RUSTUP_HOME/toolchains" ]; then + echo "sdk-matrix: seeding rustup into the cache volume (first run only)" >&2 + mkdir -p "$RUSTUP_HOME" + cp -a /opt/rust/rustup/. "$RUSTUP_HOME/" +fi +export GEM_HOME="$HOME/.gem" +export BUNDLE_PATH="$GEM_HOME" +export PATH="$GEM_HOME/bin:$HOME/.local/bin:$PATH" +export GOPATH="$HOME/go" +export NUGET_PACKAGES="$HOME/.nuget/packages" + +# Optional registry-mirror configuration forwarded by sdk-matrix-docker.sh. +# Files arrive read-only under /etc/sdk-matrix; copy them to where each tool +# looks, and remove stale copies when a file is no longer provided. +install_cfg() { # + local src="/etc/sdk-matrix/$1" dest="$2" + if [ -f "$src" ]; then + mkdir -p "$(dirname "$dest")" + cp "$src" "$dest" + chmod 0600 "$dest" + echo "sdk-matrix: using $1 override" >&2 + else + rm -f "$dest" + fi +} +install_cfg npmrc "$HOME/.npmrc" +install_cfg cargo-config.toml "$CARGO_HOME/config.toml" +install_cfg pip.conf "$HOME/.config/pip/pip.conf" +install_cfg uv.toml "$HOME/.config/uv/uv.toml" +install_cfg NuGet.Config "$HOME/.nuget/NuGet/NuGet.Config" +install_cfg gemrc "$HOME/.gemrc" +install_cfg bundle-config "$HOME/.bundle/config" + +git config --global --add safe.directory '*' >/dev/null 2>&1 || true + +cd "$WORK" +if [ ! -d node_modules ]; then + echo "sdk-matrix: installing harness dependencies (npm ci)" >&2 + npm ci +fi + +exec node scripts/sdk-matrix.mjs --cache-dir "$CACHE/sdk-under-test" "$@" diff --git a/package.json b/package.json index af723a17..10a7a008 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "lint:fix_check": "npm run lint:fix && git diff --exit-code --quiet", "tier-check": "node dist/index.js tier-check", "traceability": "tsx src/index.ts traceability", + "sdk-matrix": "node scripts/sdk-matrix.mjs", "sync-schema": "tsx scripts/sync-schema.ts", "check": "npm run typecheck && npm run lint", "typecheck": "tsgo --noEmit", diff --git a/scripts/sdk-matrix-docker.sh b/scripts/sdk-matrix-docker.sh new file mode 100755 index 00000000..620d55c7 --- /dev/null +++ b/scripts/sdk-matrix-docker.sh @@ -0,0 +1,115 @@ +#!/usr/bin/env bash +# Run scripts/sdk-matrix.mjs inside the docker/sdk-matrix toolchain image, so +# every SDK in KNOWN_SDKS can be built locally without installing six +# toolchains. Builds the image on first use; SDK clones, builds and package +# caches persist in a named volume so reruns are fast. +# +# scripts/sdk-matrix-docker.sh --mode client --scenario auth/metadata-default +# scripts/sdk-matrix-docker.sh --sdks rust-sdk,csharp-sdk --mode server --scenario tools-list +# scripts/sdk-matrix-docker.sh --ref 488 --mode client --suite auth +# +# Everything after the wrapper's own options is passed to sdk-matrix.mjs +# (see `node scripts/sdk-matrix.mjs --help`). Results land in +# ./sdk-matrix-results unless you pass -o (paths are relative to the checkout, +# which is mounted at /work). +# +# Wrapper options (must come first): +# --rebuild Rebuild the image even if it exists +# --image NAME Image tag (default: conformance-sdk-matrix) +# --volume NAME Cache volume (default: conformance-sdk-matrix-cache) +# --docker-arg ARG Extra `docker run` argument (repeatable) +# +# Registry mirrors (all optional; the default is each ecosystem's public +# registry). Each flag copies one config file into the container for that run: +# --npmrc FILE npm/pnpm .npmrc +# --cargo-config FILE cargo config.toml (e.g. [source.crates-io] replace-with) +# --pip-conf FILE pip.conf --uv-config FILE uv.toml +# --nuget-config FILE NuGet.Config --gemrc FILE .gemrc +# --bundle-config FILE bundler config +# --env NAME[=VALUE] Forward an environment variable (repeatable), e.g. +# --env GOPROXY --env UV_INDEX_URL --env NPM_CONFIG_REGISTRY +# +# The container gets outbound network only (no published ports) and no +# credentials: this script never mounts SSH keys, git credential helpers, gh +# config or tokens. If a mirror needs authentication, pass exactly the config +# file or variable that carries it, knowing it will be visible to the SDK +# builds inside the container. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +repo="$(cd "$here/.." && pwd)" + +image="${SDK_MATRIX_IMAGE:-conformance-sdk-matrix}" +volume="${SDK_MATRIX_VOLUME:-conformance-sdk-matrix-cache}" +rebuild=0 +docker_args=() +cfg_mounts=() + +add_cfg() { # + local file="$2" + if [ ! -f "$file" ]; then + echo "$1: no such file: $file" >&2 + exit 2 + fi + file="$(cd "$(dirname "$file")" && pwd)/$(basename "$file")" + cfg_mounts+=(-v "$file:/etc/sdk-matrix/$3:ro") +} + +while [ $# -gt 0 ]; do + case "$1" in + --rebuild) rebuild=1; shift ;; + --image) image="$2"; shift 2 ;; + --volume) volume="$2"; shift 2 ;; + --docker-arg) docker_args+=("$2"); shift 2 ;; + --npmrc) add_cfg "$1" "$2" npmrc; shift 2 ;; + --cargo-config) add_cfg "$1" "$2" cargo-config.toml; shift 2 ;; + --pip-conf) add_cfg "$1" "$2" pip.conf; shift 2 ;; + --uv-config) add_cfg "$1" "$2" uv.toml; shift 2 ;; + --nuget-config) add_cfg "$1" "$2" NuGet.Config; shift 2 ;; + --gemrc) add_cfg "$1" "$2" gemrc; shift 2 ;; + --bundle-config) add_cfg "$1" "$2" bundle-config; shift 2 ;; + --env) docker_args+=(-e "$2"); shift 2 ;; + --) shift; break ;; + *) break ;; + esac +done + +if ! command -v docker >/dev/null 2>&1; then + echo "docker is required" >&2 + exit 2 +fi + +if [ "$rebuild" = 1 ] || ! docker image inspect "$image" >/dev/null 2>&1; then + echo "Building $image (this takes a few minutes the first time)..." >&2 + docker build -t "$image" \ + --build-arg "UID=$(id -u)" --build-arg "GID=$(id -g)" \ + "$repo/docker/sdk-matrix" +fi + +# Label the run with the checkout's ref: inside the container a worktree's +# .git may not resolve, so read it here. +ref="$(git -C "$repo" rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)" +sha="$(git -C "$repo" rev-parse --short=12 HEAD 2>/dev/null || echo unknown)" + +tty_args=() +if [ -t 0 ] && [ -t 1 ]; then tty_args=(-it); fi + +# Rootless Docker maps the invoking host user to uid 0 inside the container, so +# there the image's uid-matched user would NOT own the bind mount; run as +# container-root instead (which is the unprivileged host user). +user_args=() +if docker info -f '{{.SecurityOptions}}' 2>/dev/null | grep -q rootless; then + user_args=(--user 0:0) +fi + +name="conformance-sdk-matrix-$$" +echo "Running in container $name (volume $volume); results under $repo/sdk-matrix-results unless -o is given" >&2 +# ${arr[@]+"${arr[@]}"}: empty-array-safe expansion for bash 3.2 (macOS) under set -u. +exec docker run --rm --name "$name" ${tty_args[@]+"${tty_args[@]}"} ${user_args[@]+"${user_args[@]}"} \ + -v "$repo:/work" \ + -v "$volume:/cache" \ + -e "SDK_MATRIX_HARNESS_REF=$ref" \ + -e "SDK_MATRIX_HARNESS_SHA=$sha" \ + ${cfg_mounts[@]+"${cfg_mounts[@]}"} \ + ${docker_args[@]+"${docker_args[@]}"} \ + "$image" "$@" diff --git a/scripts/sdk-matrix.mjs b/scripts/sdk-matrix.mjs new file mode 100644 index 00000000..1a70e23f --- /dev/null +++ b/scripts/sdk-matrix.mjs @@ -0,0 +1,1612 @@ +#!/usr/bin/env node +// Run one conformance selection (scenario / suite / requirement set) across +// every SDK in KNOWN_SDKS and render an SDK x check matrix. +// +// This is deliberately thin orchestration over `conformance sdk`: cloning, +// building and running each SDK is that command's job. This script only fans +// out over SDKs (one SDK's failure never stops the others), captures each +// run's log and the checks.json files it writes, and aggregates them into +// matrix.json + matrix.md. `--merge` re-renders from previously written +// matrix.json files, which is how the CI report job combines per-SDK legs. +// +// Usage: node scripts/sdk-matrix.mjs --mode client --scenario auth/metadata-default +// node scripts/sdk-matrix.mjs --help + +import { spawn, spawnSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +export const REPO_ROOT = path.resolve(HERE, '..'); +const DEFAULT_HARNESS_REPO = + 'https://github.com/modelcontextprotocol/conformance.git'; + +// --------------------------------------------------------------------------- +// CLI parsing + +const HELP = `Usage: node scripts/sdk-matrix.mjs [options] + +Runs \`conformance sdk \` for each SDK and aggregates the results into +/matrix.json and /matrix.md (also printed to stdout). + +Selection: + --sdks SDKs to run: KNOWN_SDKS names, optionally + name@ref (default: all) + --mode Side to test (default: client) + --scenario Scenario(s) to run (one sdk invocation each) + --suite Suite to run instead of scenarios + --requirements Requirement set to run instead (e.g. 2026-07-28) + --spec-version Passed through to \`conformance sdk\` + --timeout Passed through to \`conformance sdk\` + +Harness: + --ref Conformance ref to test. A bare number is a PR + (fetched as pull//head). Default: this + checkout, rebuilt first. + --harness-repo Where --ref is fetched from (default: upstream) + --harness-dir Use this already-built conformance checkout as + the harness (its dist/ and KNOWN_SDKS) instead + of this one; nothing is rebuilt + --skip-harness-build Don't rebuild this checkout before running + --skip-build Reuse each SDK's previous build (passed through) + --cache-dir SDK clone/build cache (default: .sdk-under-test) + --concurrency SDKs in flight at once (default: 2). Server-mode + runs are serialized regardless, because every + SDK's conformance server listens on port 3000. + +Output: + -o, --output Result directory (default: sdk-matrix-results) + --title Heading for matrix.md + --merge Run nothing; merge the matrix.json files found + under these directories and re-render + --list-sdks [--json] Print the KNOWN_SDKS names (of --ref, if given) + --strict Exit 1 if the run would turn any SDK's own CI + red: a failure its baseline does not excuse, a + stale baseline entry, or an SDK that errored + --strict-errors Exit 1 only if an SDK could not be built or run + -h, --help +`; + +export function parseArgs(argv) { + const opts = { + sdks: 'all', + mode: 'client', + scenario: undefined, + suite: undefined, + requirements: undefined, + specVersion: undefined, + timeout: undefined, + ref: undefined, + harnessRepo: DEFAULT_HARNESS_REPO, + harnessDir: undefined, + skipHarnessBuild: false, + skipBuild: false, + cacheDir: undefined, + concurrency: 2, + output: 'sdk-matrix-results', + title: undefined, + merge: [], + listSdks: false, + json: false, + strict: false, + strictErrors: false, + help: false + }; + const takesValue = { + '--sdks': 'sdks', + '--mode': 'mode', + '--scenario': 'scenario', + '--suite': 'suite', + '--requirements': 'requirements', + '--spec-version': 'specVersion', + '--timeout': 'timeout', + '--ref': 'ref', + '--pr': 'ref', + '--harness-repo': 'harnessRepo', + '--harness-dir': 'harnessDir', + '--cache-dir': 'cacheDir', + '--concurrency': 'concurrency', + '-o': 'output', + '--output': 'output', + '--title': 'title', + '--merge': 'merge' + }; + const flags = { + '--skip-harness-build': 'skipHarnessBuild', + '--skip-build': 'skipBuild', + '--list-sdks': 'listSdks', + '--json': 'json', + '--strict': 'strict', + '--strict-errors': 'strictErrors', + '-h': 'help', + '--help': 'help' + }; + for (let i = 0; i < argv.length; i++) { + const arg = argv[i]; + const eq = arg.startsWith('--') ? arg.indexOf('=') : -1; + const key = eq > 0 ? arg.slice(0, eq) : arg; + const inline = eq > 0 ? arg.slice(eq + 1) : undefined; + if (key in takesValue) { + const value = inline ?? argv[++i]; + if (value === undefined || value === '' || value.startsWith('--')) { + // Empty values come from unset workflow inputs; treat as "not given". + if (value === '') continue; + throw new Error(`${key} requires a value`); + } + if (key === '--merge') opts.merge.push(...splitList(value)); + else if (key === '--concurrency') opts.concurrency = Number(value); + else opts[takesValue[key]] = value; + } else if (key in flags) { + opts[flags[key]] = true; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + if (!['client', 'server', 'both'].includes(opts.mode)) { + throw new Error(`--mode must be client, server or both (got ${opts.mode})`); + } + const selections = [opts.scenario, opts.suite, opts.requirements].filter( + (v) => v !== undefined + ); + if (selections.length > 1) { + throw new Error('Pass at most one of --scenario, --suite, --requirements'); + } + if (!Number.isInteger(opts.concurrency) || opts.concurrency < 1) { + throw new Error('--concurrency must be a positive integer'); + } + return opts; +} + +export function splitList(value) { + return String(value) + .split(',') + .map((s) => s.trim()) + .filter(Boolean); +} + +// --------------------------------------------------------------------------- +// KNOWN_SDKS discovery +// +// The SDK list comes from the harness under test (which may be an older ref +// that predates this script), so it is read from that checkout's source rather +// than imported. KNOWN_SDKS is a prettier-formatted object literal whose +// top-level keys sit at two-space indent; the unit test cross-checks this +// parse against the real module so a format change can't silently drift. + +export function parseKnownSdkNames(source) { + const start = source.indexOf('export const KNOWN_SDKS'); + if (start < 0) throw new Error('KNOWN_SDKS not found in known-sdks.ts'); + const names = []; + for (const m of source.slice(start).matchAll(/^ {2}'([^']+)':\s*\{/gm)) { + names.push(m[1]); + } + if (names.length === 0) { + throw new Error('No SDK entries parsed from KNOWN_SDKS'); + } + return names; +} + +export function listKnownSdks(harnessRoot) { + const file = path.join(harnessRoot, 'src', 'sdk-runner', 'known-sdks.ts'); + return parseKnownSdkNames(fs.readFileSync(file, 'utf-8')); +} + +/** `name[@ref]` -> { spec, name, ref }. Mirrors parseSdkSpec in checkout.ts. */ +export function parseSdkSpec(spec) { + const at = spec.lastIndexOf('@'); + if (at <= 0) return { spec, name: spec, ref: undefined }; + const ref = spec.slice(at + 1) || undefined; + return { spec, name: spec.slice(0, at), ref }; +} + +/** The KNOWN_SDKS key a spec resolves to (basename of owner/repo). */ +export function sdkKey(name) { + return name.split('/').pop(); +} + +export function resolveSdkList(sdksArg, known) { + if (!sdksArg || sdksArg === 'all') return known.map((k) => parseSdkSpec(k)); + return splitList(sdksArg).map((s) => parseSdkSpec(s)); +} + +/** Filesystem/artifact-safe form of an SDK spec. */ +export function safeName(spec) { + return spec.replace(/[^A-Za-z0-9._-]+/g, '_'); +} + +// --------------------------------------------------------------------------- +// Toolchain probes, reported per SDK so a red cell can be read against the +// toolchain that produced it. + +const PROBES = { + node: ['node', ['--version']], + npm: ['npm', ['--version']], + pnpm: ['pnpm', ['--version']], + uv: ['uv', ['--version']], + python: ['python3', ['--version']], + go: ['go', ['version']], + cargo: ['cargo', ['--version']], + rustc: ['rustc', ['--version']], + dotnet: ['dotnet', ['--version']], + ruby: ['ruby', ['--version']], + bundler: ['bundle', ['--version']], + java: ['java', ['-version']] +}; + +const SDK_PROBES = [ + [/typescript-sdk/, ['node', 'pnpm', 'npm']], + [/python-sdk/, ['uv', 'python']], + [/go-sdk/, ['go']], + [/rust-sdk/, ['cargo', 'rustc']], + [/csharp-sdk/, ['dotnet']], + [/ruby-sdk/, ['ruby', 'bundler']], + [/java-sdk|kotlin-sdk/, ['java']] +]; + +export function probesFor(sdkName) { + for (const [re, probes] of SDK_PROBES) { + if (re.test(sdkName)) return probes; + } + return ['node']; +} + +function probeToolchain(sdkName, cwd) { + const out = {}; + for (const key of probesFor(sdkName)) { + const [cmd, args] = PROBES[key]; + try { + // Probe inside the SDK checkout when we have it, so per-repo pins + // (rust-toolchain.toml, packageManager, .python-version) are reflected. + const r = spawnSync(cmd, args, { + cwd: cwd && fs.existsSync(cwd) ? cwd : undefined, + encoding: 'utf-8', + timeout: 120_000, + env: { ...process.env, COREPACK_ENABLE_DOWNLOAD_PROMPT: '0' } + }); + if (r.error || r.status !== 0) { + out[key] = null; + continue; + } + const text = `${r.stdout || ''}\n${r.stderr || ''}`.trim(); + out[key] = text.split('\n')[0].trim() || null; + } catch { + out[key] = null; + } + } + return out; +} + +// --------------------------------------------------------------------------- +// Process helpers + +function run(cmd, args, { cwd, logFile, prefix, env } = {}) { + return new Promise((resolve) => { + const started = Date.now(); + const log = logFile ? fs.createWriteStream(logFile, { flags: 'a' }) : null; + if (log) log.write(`$ ${cmd} ${args.join(' ')}\n`); + const child = spawn(cmd, args, { + cwd, + env: { ...process.env, ...env }, + stdio: ['ignore', 'pipe', 'pipe'] + }); + let captured = ''; + let partial = ''; + const onData = (chunk) => { + const text = chunk.toString(); + captured += text; + if (captured.length > 4_000_000) captured = captured.slice(-2_000_000); + if (log) log.write(text); + if (prefix !== undefined) { + const pieces = (partial + text).split('\n'); + partial = pieces.pop() ?? ''; + for (const line of pieces) process.stderr.write(`${prefix}${line}\n`); + } + }; + child.stdout.on('data', onData); + child.stderr.on('data', onData); + child.on('error', (err) => { + captured += `\nspawn error: ${err.message}\n`; + if (log) log.end(`\nspawn error: ${err.message}\n`); + resolve({ + exitCode: -1, + output: captured, + durationMs: Date.now() - started + }); + }); + child.on('close', (code) => { + if (prefix !== undefined && partial) { + process.stderr.write(`${prefix}${partial}\n`); + } + if (log) log.end(`\n[exit ${code}]\n`); + resolve({ + exitCode: code ?? -1, + output: captured, + durationMs: Date.now() - started + }); + }); + }); +} + +function git(args, cwd) { + const r = spawnSync('git', args, { cwd, encoding: 'utf-8' }); + if (r.status !== 0) { + throw new Error(`git ${args.join(' ')} failed: ${r.stderr || r.stdout}`); + } + return r.stdout.trim(); +} + +function tryGit(args, cwd) { + try { + return git(args, cwd) || undefined; + } catch { + return undefined; + } +} + +/** Promise pool: run `fn` over items with at most `n` in flight. */ +export async function pool(items, n, fn) { + const results = new Array(items.length); + let next = 0; + const workers = Array.from( + { length: Math.min(n, items.length) }, + async () => { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i], i); + } + } + ); + await Promise.all(workers); + return results; +} + +/** A minimal async mutex. */ +function createLock() { + let tail = Promise.resolve(); + return (fn) => { + const runIt = tail.then(fn, fn); + tail = runIt.catch(() => {}); + return runIt; + }; +} + +// --------------------------------------------------------------------------- +// Harness preparation (--ref) + +async function prepareHarness(opts, cacheDir, outDir) { + if (opts.harnessDir) { + // An existing, already-built checkout (CI builds the ref under test in a + // separate directory and drives it with this branch's script). + const root = path.resolve(opts.harnessDir); + if (!opts.listSdks && !fs.existsSync(path.join(root, 'dist', 'index.js'))) { + throw new Error( + `No dist/index.js in --harness-dir ${root}; run npm ci && npm run build there` + ); + } + return { root, ...describeCheckout(root) }; + } + if (!opts.ref) { + const root = REPO_ROOT; + if (!opts.skipHarnessBuild) { + console.error('[matrix] Building harness (npm run build)'); + const logFile = path.join(outDir, 'harness-build.log'); + const r = await run('npm', ['run', 'build', '--silent'], { + cwd: root, + logFile + }); + if (r.exitCode !== 0) { + throw new Error(`Harness build failed (see ${logFile})`); + } + } + if (!fs.existsSync(path.join(root, 'dist', 'index.js'))) { + throw new Error(`No dist/index.js in ${root}; run npm run build`); + } + return { root, ...describeCheckout(root) }; + } + + // A separate clone (not a worktree of this checkout) so this also works + // where the invoking checkout has no usable .git, e.g. a bind-mounted + // worktree inside the container. + const isPr = /^\d+$/.test(opts.ref); + const refspec = isPr ? `pull/${opts.ref}/head` : opts.ref; + const root = path.join(cacheDir, '_harness', 'conformance'); + fs.mkdirSync(path.dirname(root), { recursive: true }); + if (!fs.existsSync(path.join(root, '.git'))) { + console.error(`[matrix] Cloning ${opts.harnessRepo} -> ${root}`); + git(['clone', opts.harnessRepo, root], path.dirname(root)); + } + console.error(`[matrix] Fetching ${refspec} from ${opts.harnessRepo}`); + git(['fetch', opts.harnessRepo, refspec], root); + const sha = git(['rev-parse', 'FETCH_HEAD'], root); + git(['checkout', '--detach', '--force', sha], root); + git(['clean', '-fdx', '-e', 'node_modules', '-e', 'dist'], root); + + const logFile = path.join(outDir, 'harness-build.log'); + const lock = fs.readFileSync(path.join(root, 'package-lock.json'), 'utf-8'); + const lockStamp = `${lock.length}:${simpleHash(lock)}`; + const stampFile = path.join(root, 'node_modules', '.sdk-matrix-lock'); + const prev = fs.existsSync(stampFile) + ? fs.readFileSync(stampFile, 'utf-8') + : ''; + if (prev !== lockStamp) { + console.error('[matrix] Installing harness dependencies (npm ci)'); + const r = await run('npm', ['ci'], { cwd: root, logFile }); + if (r.exitCode !== 0) { + throw new Error(`npm ci failed for --ref ${opts.ref} (see ${logFile})`); + } + fs.writeFileSync(stampFile, lockStamp); + } + console.error('[matrix] Building harness at ref (npm run build)'); + const r = await run('npm', ['run', 'build', '--silent'], { + cwd: root, + logFile + }); + if (r.exitCode !== 0) { + throw new Error( + `Harness build failed for --ref ${opts.ref} (see ${logFile})` + ); + } + return { + root, + ref: isPr ? `PR #${opts.ref}` : opts.ref, + sha: sha.slice(0, 12), + version: readVersion(root) + }; +} + +function simpleHash(text) { + let h = 0; + for (let i = 0; i < text.length; i++) h = (h * 31 + text.charCodeAt(i)) | 0; + return (h >>> 0).toString(16); +} + +function readVersion(root) { + try { + return JSON.parse(fs.readFileSync(path.join(root, 'package.json'), 'utf-8')) + .version; + } catch { + return undefined; + } +} + +function describeCheckout(root) { + // Env overrides let a wrapper (the docker script) label a checkout whose + // .git is not resolvable from where this runs. + return { + ref: + process.env.SDK_MATRIX_HARNESS_REF || + tryGit(['rev-parse', '--abbrev-ref', 'HEAD'], root) || + 'unknown', + sha: + process.env.SDK_MATRIX_HARNESS_SHA || + tryGit(['rev-parse', '--short=12', 'HEAD'], root) || + 'unknown', + version: readVersion(root) + }; +} + +// --------------------------------------------------------------------------- +// Running one SDK + +/** + * Classify a `conformance sdk` invocation from its combined output. The sdk + * command logs `[sdk] ` progress lines and, on a thrown error, a final + * `[sdk] `; `[sdk] conformance ...` is printed right + * before scenarios start, so its absence means the run never got that far. + */ +export function classifyInvocation(output, exitCode) { + if (/^\[sdk\] conformance (client|server)\b/m.test(output)) { + return { phase: 'ran', exitCode }; + } + const progress = + /^(Fetching|Cloning|Checking out|HEAD is|Building:|No build command|Starting server|Server ready|Stopping server|conformance )/; + let reason; + for (const m of output.matchAll(/^\[sdk\] (.+)$/gm)) { + if (!progress.test(m[1])) reason = m[1]; + } + let phase = 'setup'; + if (/^\[sdk\] Starting server/m.test(output)) phase = 'server-start'; + else if (/^\[sdk\] Building:/m.test(output)) phase = 'build'; + else if (/^\[sdk\] (Cloning|Fetching|Checking out)/m.test(output)) { + phase = 'checkout'; + } + return { + phase, + exitCode, + reason: reason ?? `exited with code ${exitCode} before running scenarios`, + detail: firstErrorLine(output) + }; +} + +/** First line that looks like a toolchain error, for the one-line summary. */ +export function firstErrorLine(output) { + const patterns = [ + /failed to select a version for the requirement.*$/m, + /^error(\[E\d+\])?: .+$/m, + /^npm (ERR!|error) .+$/m, + /ERR_PNPM_[A-Z_]+.*$/m, + /^.*error (CS|MSB|NU|NETSDK)\d+: .+$/m, + /^.*(command not found|not found in PATH|No such file or directory).*$/m, + /^.*Could not (find|locate) .+$/m, + /^\s*(E|e)rror:? .+$/m + ]; + for (const re of patterns) { + const m = output.match(re); + if (m) return m[0].trim().slice(0, 240); + } + return undefined; +} + +export function tailLines(text, n) { + const lines = String(text).replace(/\r/g, '').split('\n'); + while (lines.length && !lines[lines.length - 1].trim()) lines.pop(); + return lines.slice(-n).join('\n'); +} + +/** + * Collect checks.json files written under one mode's output dir into + * { scenarioName: { checks, resultDir } }. Result dirs are named + * `-` (server mode: `server--`), and a + * scenario name containing '/' nests directories. When a scenario ran more + * than once the latest timestamp wins. + */ +export function collectModeResults(modeDir, mode) { + const found = {}; + if (!fs.existsSync(modeDir)) return {}; + const walk = (dir) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(full); + continue; + } + if (entry.name !== 'checks.json') continue; + let rel = path + .relative(modeDir, path.dirname(full)) + .split(path.sep) + .join('/'); + const ts = rel.match(/-(\d{4}-\d{2}-\d{2}T[\d-]+Z)$/); + const stamp = ts ? ts[1] : ''; + if (ts) rel = rel.slice(0, -ts[0].length); + if (mode === 'server' && rel.startsWith('server-')) { + rel = rel.slice('server-'.length); + } + let checks; + try { + checks = JSON.parse(fs.readFileSync(full, 'utf-8')); + if (!Array.isArray(checks)) throw new Error('not an array'); + } catch (err) { + checks = [ + { + id: rel, + name: rel, + status: 'FAILURE', + description: 'checks.json could not be parsed', + errorMessage: String(err) + } + ]; + } + if (!found[rel] || found[rel].stamp < stamp) { + found[rel] = { stamp, resultDir: path.dirname(full), checks }; + } + } + }; + walk(modeDir); + const out = {}; + for (const [name, v] of Object.entries(found)) { + out[name] = { resultDir: v.resultDir, checks: v.checks.map(slimCheck) }; + } + return out; +} + +function slimCheck(c) { + const slim = { + id: String(c.id ?? ''), + status: String(c.status ?? 'UNKNOWN') + }; + if (c.name && c.name !== c.id) slim.name = String(c.name); + if (c.description) slim.description = String(c.description); + if (c.errorMessage) slim.errorMessage = String(c.errorMessage); + return slim; +} + +function checkoutDirFromLog(output) { + const m = + output.match(/^\[sdk\] Cloning \S+ -> (.+)$/m) || + output.match(/^\[sdk\] Fetching \S+ \(cached at (.+)\)$/m); + return m ? m[1].trim() : undefined; +} + +function headFromLog(output) { + const m = output.match(/^\[sdk\] HEAD is (\S+)/m); + return m ? m[1] : undefined; +} + +function expectedFailuresFromLog(output) { + const m = output.match( + /^\[sdk\] conformance (?:client|server) .*--expected-failures (\S+)/m + ); + return m ? m[1] : undefined; +} + +/** + * Parse an expected-failures YAML file into { client: [...], server: [...] } + * entry strings ('' or ':'). Uses the `yaml` + * package from the harness checkout when it is installed there, else a small + * parser that covers the block-list shape these files use. + */ +export function parseBaselineYaml(text, yamlParse) { + const norm = (list) => + Array.isArray(list) + ? list + .filter((e) => typeof e === 'string' && e.trim()) + .map((e) => e.trim()) + : []; + if (yamlParse) { + try { + const doc = yamlParse(text) ?? {}; + if (doc && typeof doc === 'object' && !Array.isArray(doc)) { + return { client: norm(doc.client), server: norm(doc.server) }; + } + } catch { + // fall through to the minimal parser + } + } + const out = { client: [], server: [] }; + let section = null; + for (const raw of text.split('\n')) { + const line = raw.replace(/\s+#.*$/, '').replace(/^#.*$/, ''); + if (!line.trim()) continue; + const key = line.match(/^([A-Za-z_]+):\s*(\[(.*)\])?\s*$/); + if (key) { + section = key[1] in out ? key[1] : null; + if (section && key[2]) { + out[section].push( + ...key[3] + .split(',') + .map((s) => s.trim().replace(/^['"]|['"]$/g, '')) + .filter(Boolean) + ); + } + continue; + } + const item = line.match(/^\s*-\s+(.+?)\s*$/); + if (item && section) { + out[section].push(item[1].replace(/^['"]|['"]$/g, '')); + } + } + return out; +} + +function loadYamlParse(harnessRoot) { + for (const base of [harnessRoot, REPO_ROOT]) { + try { + const req = createRequire(path.join(base, 'package.json')); + return req('yaml').parse; + } catch { + // not installed there + } + } + return undefined; +} + +function readBaseline(file, mode, harnessRoot) { + try { + const text = fs.readFileSync(file, 'utf-8'); + const parsed = parseBaselineYaml(text, loadYamlParse(harnessRoot)); + return { file, entries: parsed[mode] ?? [] }; + } catch (err) { + return { file, entries: [], error: String(err.message ?? err) }; + } +} + +async function runSdk(sdk, ctx) { + const { opts, harness, cacheDir, outDir, serverLock } = ctx; + const sdkDir = path.join(outDir, 'sdks', safeName(sdk.spec)); + fs.mkdirSync(sdkDir, { recursive: true }); + const record = { + spec: sdk.spec, + name: sdk.name, + requestedRef: sdk.ref ?? null, + head: null, + checkoutDir: null, + toolchain: {}, + modes: {} + }; + const modes = opts.mode === 'both' ? ['client', 'server'] : [opts.mode]; + let built = opts.skipBuild; + let buildError = null; + + for (const mode of modes) { + const modeOut = path.join(sdkDir, mode); + const modeRec = { + invocations: [], + error: null, + baseline: null, + scenarios: {} + }; + record.modes[mode] = modeRec; + let baselineFile; + if (buildError) { + // The build already failed under the previous mode; don't repeat it. + modeRec.error = { ...buildError, inherited: true }; + continue; + } + // `conformance sdk --scenario` takes one scenario, so a list means one + // invocation each; everything after the first reuses the build. + const selections = opts.scenario ? splitList(opts.scenario) : [null]; + for (const scenario of selections) { + const args = [ + path.join(harness.root, 'dist', 'index.js'), + 'sdk', + sdk.spec, + '--mode', + mode, + '--cache-dir', + cacheDir, + '-o', + modeOut + ]; + if (scenario) args.push('--scenario', scenario); + else if (opts.suite) args.push('--suite', opts.suite); + else if (opts.requirements) + args.push('--requirements', opts.requirements); + if (opts.specVersion) args.push('--spec-version', opts.specVersion); + if (opts.timeout) args.push('--timeout', opts.timeout); + if (built) args.push('--skip-build'); + const label = scenario ?? opts.suite ?? opts.requirements ?? 'default'; + const logFile = path.join(sdkDir, `${mode}-${safeName(label)}.log`); + console.error(`[matrix] ${sdk.spec} ${mode} ${label}: starting`); + const exec = () => + run(process.execPath, args, { + cwd: harness.root, + logFile, + prefix: `[${sdk.spec}] `, + env: { COREPACK_ENABLE_DOWNLOAD_PROMPT: '0' } + }); + // Every SDK's conformance server binds port 3000, so server-mode runs + // never overlap across SDKs; client-mode mock servers use ephemeral + // ports and run concurrently. + const r = mode === 'server' ? await serverLock(exec) : await exec(); + record.checkoutDir ??= checkoutDirFromLog(r.output) ?? null; + record.head ??= headFromLog(r.output) ?? null; + baselineFile ??= expectedFailuresFromLog(r.output); + const cls = classifyInvocation(r.output, r.exitCode); + const inv = { + scenario: scenario ?? null, + args: args.slice(1), + exitCode: r.exitCode, + durationMs: r.durationMs, + phase: cls.phase, + logFile: path.relative(outDir, logFile) + }; + if (cls.phase !== 'ran') { + inv.reason = cls.reason; + if (cls.detail) inv.detail = cls.detail; + inv.tail = tailLines(r.output, 40); + modeRec.error ??= { + phase: cls.phase, + reason: cls.reason, + detail: cls.detail ?? null, + logFile: inv.logFile, + tail: inv.tail + }; + if (cls.phase !== 'server-start') buildError = modeRec.error; + } else { + built = true; + } + modeRec.invocations.push(inv); + console.error( + `[matrix] ${sdk.spec} ${mode} ${label}: ${cls.phase} (exit ${r.exitCode}, ${Math.round(r.durationMs / 1000)}s)` + ); + if (buildError) break; + } + modeRec.scenarios = collectModeResults(modeOut, mode); + // The SDK's own expected-failures baseline (the file `conformance sdk` + // passed as --expected-failures) is what separates "this change breaks + // that SDK's CI" from "that SDK already knows it fails this". + if (baselineFile) { + modeRec.baseline = readBaseline(baselineFile, mode, harness.root); + if (record.checkoutDir && modeRec.baseline) { + modeRec.baseline.file = path.relative(record.checkoutDir, baselineFile); + } + } + // A requested scenario that left no checks.json is an execution error + // for that scenario, distinct from a check that was simply not emitted. + if (opts.scenario) { + for (const s of splitList(opts.scenario)) { + if (modeRec.scenarios[s]) continue; + const inv = modeRec.invocations.find((i) => i.scenario === s); + modeRec.scenarios[s] = { + resultDir: null, + checks: [], + missing: true, + reason: + inv?.detail ?? + inv?.reason ?? + modeRec.error?.reason ?? + (inv ? `no checks.json written (exit ${inv.exitCode})` : 'not run') + }; + } + } + } + record.toolchain = probeToolchain(sdk.name, record.checkoutDir ?? undefined); + fs.writeFileSync( + path.join(sdkDir, 'result.json'), + JSON.stringify(record, null, 2) + ); + return record; +} + +// --------------------------------------------------------------------------- +// Aggregation + rendering + +const STATUS_RANK = { + FAILURE: 5, + WARNING: 4, + SUCCESS: 3, + INFO: 2, + SKIPPED: 1 +}; +const ICON = { + SUCCESS: '✅', + FAILURE: '❌', + WARNING: '⚠️', + SKIPPED: '⏭️', + INFO: 'ℹ️', + // Fails, but the SDK's own expected-failures baseline already lists it. + BASELINED: '⭕', + STALE: '\u{1F9F9}' +}; +const DASH = '—'; + +export function worstStatus(statuses) { + let worst; + for (const s of statuses) { + if ( + worst === undefined || + (STATUS_RANK[s] ?? 0) > (STATUS_RANK[worst] ?? 0) + ) { + worst = s; + } + } + return worst; +} + +export function summarizeChecks(checks) { + const n = (s) => checks.filter((c) => c.status === s).length; + // Denominator matches the harness's own summary: SUCCESS + FAILURE. + // WARNING/INFO/SKIPPED are reported alongside, not scored. + return { + passed: n('SUCCESS'), + failed: n('FAILURE'), + warnings: n('WARNING'), + total: n('SUCCESS') + n('FAILURE'), + emitted: checks.length + }; +} + +const isFailing = (status) => status === 'FAILURE' || status === 'WARNING'; + +/** Baseline lookup for one mode record ({ has, scenarios, checks }). */ +function baselineIndex(modeRec) { + const entries = modeRec?.baseline?.entries ?? []; + const scenarios = new Set(); + const checks = new Set(); + for (const e of entries) { + if (e.includes(':')) checks.add(e); + else scenarios.add(e); + } + return { has: Boolean(modeRec?.baseline), scenarios, checks }; +} + +function isBaselined(idx, scenario, checkId) { + return ( + idx.scenarios.has(scenario) || idx.checks.has(`${scenario}:${checkId}`) + ); +} + +/** + * Judge one scenario result against the SDK's own expected-failures baseline, + * mirroring evaluateBaseline in src/expected-failures.ts (FAILURE and WARNING + * both count as failing). `unexpected` are failing checks the baseline does not + * excuse: those turn that SDK's CI red, i.e. regressions from its point of + * view. `baselined` are failing checks it already expects. `stale` are baseline + * entries for this scenario that no longer fail, which also turns its CI red + * until the entry is removed. + */ +export function judgeScenario(modeRec, scenario) { + const out = { unexpected: [], baselined: [], stale: [] }; + const r = modeRec?.scenarios?.[scenario]; + if (!r || r.missing) return out; + const idx = baselineIndex(modeRec); + const failing = r.checks.filter((c) => isFailing(c.status)); + for (const c of failing) { + (isBaselined(idx, scenario, c.id) ? out.baselined : out.unexpected).push(c); + } + if (idx.scenarios.has(scenario) && failing.length === 0) { + out.stale.push(scenario); + } + for (const e of idx.checks) { + const i = e.indexOf(':'); + if (e.slice(0, i) !== scenario) continue; + const id = e.slice(i + 1); + const present = r.checks.filter((c) => c.id === id); + if (present.length && !present.some((c) => isFailing(c.status))) { + out.stale.push(e); + } + } + return out; +} + +/** + * Everything in the matrix that would turn some SDK's own CI red (or could + * not be determined): unexpected failures, stale baseline entries, and SDKs + * that could not be built or run. + */ +export function findRegressions(matrix) { + const unexpected = []; + const stale = []; + const errors = []; + const noBaseline = []; + for (const s of Object.values(matrix.sdks)) { + for (const [mode, m] of Object.entries(s.modes)) { + if (m.error) { + errors.push({ sdk: s.spec, mode, error: m.error }); + if (m.error.inherited || Object.keys(m.scenarios).length === 0) + continue; + } + const ran = Object.values(m.scenarios).some((sc) => !sc.missing); + if (ran && !m.baseline) noBaseline.push({ sdk: s.spec, mode }); + for (const sc of Object.keys(m.scenarios).sort()) { + if (m.scenarios[sc].missing) { + errors.push({ + sdk: s.spec, + mode, + error: { + phase: 'run', + reason: m.scenarios[sc].reason, + scenario: sc + } + }); + continue; + } + const j = judgeScenario(m, sc); + for (const c of j.unexpected) { + unexpected.push({ sdk: s.spec, mode, scenario: sc, check: c }); + } + for (const e of j.stale) stale.push({ sdk: s.spec, mode, entry: e }); + } + } + } + return { unexpected, stale, errors, noBaseline }; +} + +/** + * Escape text for a markdown table cell. Everything rendered comes from SDK + * output or check messages, so it is treated as data: no raw HTML, no pipes, + * no line breaks, no link/image syntax, no @-mentions, bounded length. + */ +export function cell(text, max = 140) { + let s = String(text ?? '') + // eslint-disable-next-line no-control-regex + .replace(/[\x00-\x1f\x7f]+/g, ' ') + .replace(/\s+/g, ' ') + .trim(); + if (s.length > max) s = `${s.slice(0, max - 1)}…`; + return s + .replace(/\\/g, '\\\\') + .replace(/\|/g, '\\|') + .replace(//g, '>') + .replace(/\[/g, '\\[') + .replace(/\]/g, '\\]') + .replace(/`/g, "'") + .replace(/@/g, '@​'); +} + +function code(text, max = 200) { + const c = cell(text, max); + return c ? `\`${c}\`` : DASH; +} + +export function mergeMatrices(matrices) { + if (matrices.length === 0) throw new Error('Nothing to merge'); + const base = structuredClone(matrices[0]); + base.sdks = {}; + for (const m of matrices) { + for (const [k, v] of Object.entries(m.sdks ?? {})) base.sdks[k] = v; + } + base.generatedAt = new Date().toISOString(); + return base; +} + +/** matrix.json files directly in, or up to three levels under, each dir. */ +export function findMatrixFiles(dirs) { + const files = []; + const walk = (dir, depth) => { + if (!fs.existsSync(dir) || !fs.statSync(dir).isDirectory()) return; + const direct = path.join(dir, 'matrix.json'); + if (fs.existsSync(direct)) { + files.push(direct); + return; + } + if (depth >= 3) return; + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { + if (e.isDirectory()) walk(path.join(dir, e.name), depth + 1); + } + }; + for (const d of dirs) walk(path.resolve(d), 0); + return files; +} + +export function errorLabel(err) { + const what = + err.phase === 'build' + ? 'build failed' + : err.phase === 'checkout' + ? 'checkout failed' + : err.phase === 'server-start' + ? 'server failed to start' + : 'failed before running'; + return `${what}: ${err.detail ?? err.reason}`; +} + +function sdkStatusCell(s) { + const modes = Object.entries(s.modes); + const errors = modes + .filter(([, m]) => m.error && !m.error.inherited) + .map( + ([mode, m]) => + `${modes.length > 1 ? `${mode}: ` : ''}${errorLabel(m.error)}` + ); + if (errors.length) return `${ICON.FAILURE} ${cell(errors.join('; '), 220)}`; + const missing = modes.flatMap(([, m]) => + Object.values(m.scenarios).filter((sc) => sc.missing) + ).length; + const all = modes.flatMap(([, m]) => + Object.values(m.scenarios).flatMap((sc) => sc.checks) + ); + const sum = summarizeChecks(all); + let unexpected = 0; + let baselined = 0; + let stale = 0; + for (const [, m] of modes) { + for (const sc of Object.keys(m.scenarios)) { + const j = judgeScenario(m, sc); + unexpected += j.unexpected.length; + baselined += j.baselined.length; + stale += j.stale.length; + } + } + const parts = []; + if (missing) parts.push(`${missing} scenario(s) produced no results`); + if (unexpected) parts.push(`${unexpected} unexpected`); + if (stale) parts.push(`${stale} stale baseline`); + if (baselined) parts.push(`${baselined} baselined`); + const head = parts.length ? `${parts.join(', ')}; ` : ''; + const icon = + missing || unexpected ? ICON.FAILURE : stale ? ICON.WARNING : ICON.SUCCESS; + return `${icon} ${head}${sum.passed}/${sum.total} checks pass`; +} + +/** `go version go1.26.5 linux/amd64` -> `1.26.5`, `v22.1.0` -> `22.1.0`. */ +export function shortVersion(text) { + return String(text) + .replace( + /^(v|go version go|go version |cargo |rustc |ruby |Bundler version |Python |uv |openjdk version )/, + '' + ) + .replace(/ \(.*$/, '') + .replace(/ (linux|darwin|windows)\/\S+$/, '') + .replace(/^"(.*)"$/, '$1'); +} + +function toolchainCell(s) { + const parts = Object.entries(s.toolchain ?? {}).map( + ([k, v]) => `${k} ${v === null ? 'missing' : cell(shortVersion(v), 32)}` + ); + return parts.length ? parts.join(', ') : DASH; +} + +function scenarioCell(m, sc) { + if (!m) return DASH; + const r = m.scenarios[sc]; + if (m.error && (!r || r.missing || r.checks.length === 0)) { + return `${ICON.FAILURE} ${cell(errorLabel(m.error), 90)}`; + } + if (!r) return DASH; + if (r.missing) + return `${ICON.FAILURE} no results: ${cell(r.reason ?? '', 80)}`; + const sum = summarizeChecks(r.checks); + if (sum.emitted === 0) return `${DASH} 0 checks`; + const warn = sum.warnings ? ` (+${sum.warnings}${ICON.WARNING})` : ''; + const j = judgeScenario(m, sc); + if (j.unexpected.length) { + const icon = j.unexpected.some((c) => c.status === 'FAILURE') + ? ICON.FAILURE + : ICON.WARNING; + return `${icon} ${sum.passed}/${sum.total}${warn}`; + } + if (j.baselined.length) { + return `${ICON.BASELINED} ${sum.passed}/${sum.total}${warn} baselined`; + } + if (j.stale.length) { + return `${ICON.SUCCESS} ${sum.passed}/${sum.total} ${ICON.STALE}stale baseline`; + } + return `${ICON.SUCCESS} ${sum.passed}/${sum.total}`; +} + +/** Check ids across the given scenarios, in first-seen order. */ +function checkIds(sdks, mode, scenarios) { + const ids = []; + const seen = new Set(); + for (const sc of scenarios) { + for (const s of sdks) { + const r = s.modes[mode]?.scenarios?.[sc]; + if (!r) continue; + for (const c of r.checks) { + if (seen.has(c.id)) continue; + seen.add(c.id); + ids.push(c.id); + } + } + } + return ids; +} + +/** + * One SDK's statuses for a check id, per scenario that emitted it: + * [{ scenario, status }] with status = worst within that scenario. + */ +function statusesByScenario(sdk, mode, scenarios, id) { + const out = []; + const m = sdk.modes[mode]; + const idx = baselineIndex(m); + for (const sc of scenarios) { + const r = m?.scenarios?.[sc]; + if (!r || r.missing) continue; + const statuses = r.checks.filter((c) => c.id === id).map((c) => c.status); + if (statuses.length) { + const status = worstStatus(statuses); + out.push({ + scenario: sc, + status, + baselined: isFailing(status) && isBaselined(idx, sc, id) + }); + } + } + return out; +} + +/** Icon for a (status, baselined) pair in the check table. */ +function statusIcon(status, baselined) { + if (baselined && isFailing(status)) return ICON.BASELINED; + return ICON[status] ?? cell(status, 12); +} + +export function renderMarkdown(matrix) { + const sdks = Object.values(matrix.sdks); + const lines = []; + lines.push(`## ${cell(matrix.title ?? 'SDK matrix', 200)}`, ''); + const h = matrix.harness ?? {}; + const sel = matrix.selection ?? {}; + const what = sel.scenario + ? `scenario ${splitList(sel.scenario) + .map((s) => code(s)) + .join(', ')}` + : sel.suite + ? `suite ${code(sel.suite)}` + : sel.requirements + ? `requirements ${code(sel.requirements)}` + : 'default suites'; + const version = h.version ? `, v${cell(h.version, 40)}` : ''; + lines.push( + `Conformance ${code(h.ref ?? 'unknown')} (${code(h.sha ?? 'unknown')}${version}), mode ${code(sel.mode ?? '?')}, ${what}. Generated ${cell(matrix.generatedAt, 40)}${matrix.host?.runner ? ` on ${cell(matrix.host.runner, 20)}` : ''}.`, + '' + ); + lines.push( + `Legend: ${ICON.SUCCESS} SUCCESS, ${ICON.FAILURE} FAILURE and ${ICON.WARNING} WARNING not in the SDK's own expected-failures baseline (would turn its CI red), ${ICON.BASELINED} fails but baselined by the SDK (its CI stays green), ${ICON.STALE} baselined but now passes (stale entry, also turns its CI red), ${ICON.SKIPPED} SKIPPED, ${ICON.INFO} INFO, ${DASH} not emitted or not run. Summary cells are passed/(passed+failed) checks.`, + '' + ); + + lines.push( + '| SDK | SDK head | Status | Baseline | Toolchain |', + '| --- | --- | --- | --- | --- |' + ); + for (const s of sdks) { + const files = [ + ...new Set( + Object.values(s.modes) + .map((m) => m.baseline?.file) + .filter(Boolean) + ) + ]; + const baseline = files.length + ? files.map((f) => code(f, 80)).join(', ') + : 'none'; + lines.push( + `| ${code(s.spec)} | ${s.head ? code(s.head) : DASH} | ${sdkStatusCell(s)} | ${baseline} | ${toolchainCell(s)} |` + ); + } + lines.push(''); + + // The question this report exists to answer: does the harness change turn + // any SDK's own conformance CI red? That is every failing check the SDK's + // expected-failures baseline does not already excuse, plus baseline + // entries that now pass (stale), plus SDKs we could not run at all. + const reg = findRegressions(matrix); + lines.push('### Regressions', ''); + if ( + reg.unexpected.length === 0 && + reg.stale.length === 0 && + reg.errors.length === 0 + ) { + lines.push( + `${ICON.SUCCESS} None. Every failing check is already in that SDK's expected-failures baseline, and every SDK built and ran.`, + '' + ); + } else { + if (reg.unexpected.length) { + const cap = 60; + lines.push( + `${ICON.FAILURE} ${reg.unexpected.length} failing check(s) not covered by the SDK's baseline:`, + '', + '| SDK | Mode | Scenario | Check | Message |', + '| --- | --- | --- | --- | --- |', + ...reg.unexpected + .slice(0, cap) + .map( + (u) => + `| ${code(u.sdk)} | ${u.mode} | ${code(u.scenario)} | ${ICON[u.check.status]} ${code(u.check.id)} | ${cell(u.check.errorMessage ?? u.check.description ?? '', 160) || DASH} |` + ) + ); + if (reg.unexpected.length > cap) { + lines.push( + `| | | | | ${reg.unexpected.length - cap} more in matrix.json |` + ); + } + lines.push(''); + } + if (reg.stale.length) { + lines.push( + `${ICON.STALE} ${reg.stale.length} stale baseline entr${reg.stale.length === 1 ? 'y' : 'ies'} (passes now; the SDK's CI fails until the entry is removed):`, + '', + ...reg.stale.map((s) => `- ${code(s.sdk)} ${s.mode}: ${code(s.entry)}`), + '' + ); + } + if (reg.errors.length) { + lines.push( + `${ICON.FAILURE} Could not determine for:`, + '', + ...reg.errors.map( + (e) => + `- ${code(e.sdk)} ${e.mode}${e.error.scenario ? ` ${code(e.error.scenario)}` : ''}: ${cell(e.error.inherited ? 'build failed (see above)' : errorLabel(e.error), 160)}` + ), + '' + ); + } + } + if (reg.noBaseline.length) { + lines.push( + `No expected-failures baseline was applied for ${[...new Set(reg.noBaseline.map((n) => code(n.sdk)))].join(', ')} (none configured in KNOWN_SDKS, or a requirements run), so every failure there counts as unexpected.`, + '' + ); + } + + const header = `| ${sdks.map((s) => code(s.spec)).join(' | ')} |`; + const rule = `|${' --- |'.repeat(sdks.length)}`; + const modes = [...new Set(sdks.flatMap((s) => Object.keys(s.modes)))]; + for (const mode of modes) { + lines.push(`### ${mode}`, ''); + const scenarioNames = [ + ...new Set( + sdks.flatMap((s) => Object.keys(s.modes[mode]?.scenarios ?? {})) + ) + ].sort(); + if (scenarioNames.length === 0) { + const anyError = sdks.some((s) => s.modes[mode]?.error); + lines.push( + anyError + ? '_No scenario results; see errors below._' + : '_No scenario results._', + '' + ); + continue; + } + lines.push(`| Scenario ${header}`, `| --- ${rule}`); + for (const sc of scenarioNames) { + const cells = sdks.map((s) => scenarioCell(s.modes[mode], sc)); + lines.push(`| ${code(sc)} | ${cells.join(' | ')} |`); + } + lines.push(''); + + // One SDK x check table per mode. Check ids are unioned across scenarios + // (several scenarios usually emit the same ids, e.g. every auth/* flow); + // a cell is the worst status across the scenarios that emitted it, and + // any check whose result differs between scenarios is broken out below. + const ids = checkIds(sdks, mode, scenarioNames); + if (ids.length) { + const open = ids.length <= 60 ? ' open' : ''; + const differs = []; + const scope = + scenarioNames.length === 1 + ? cell(scenarioNames[0]) + : `${scenarioNames.length} scenarios`; + lines.push( + `${mode} checks: ${ids.length} (${scope})`, + '', + `| Check ${header}`, + `| --- ${rule}` + ); + for (const id of ids) { + const cells = sdks.map((s) => { + const per = statusesByScenario(s, mode, scenarioNames, id); + if (per.length === 0) return DASH; + const icons = per.map((p) => statusIcon(p.status, p.baselined)); + // Worst first: an unexcused failure anywhere wins over a baselined + // one, which wins over a pass. + const worst = + per.find((p) => isFailing(p.status) && !p.baselined) ?? + per.find((p) => isFailing(p.status)) ?? + per.find((p) => p.status === worstStatus(per.map((q) => q.status))); + const icon = statusIcon(worst.status, worst.baselined); + if (new Set(icons).size > 1) { + differs.push( + `- ${code(s.spec)} ${code(id)}: ${per.map((p, i) => `${icons[i]} ${code(p.scenario)}`).join(', ')}` + ); + return `${icon}\\*`; + } + return icon; + }); + lines.push(`| ${code(id)} | ${cells.join(' | ')} |`); + } + lines.push(''); + if (differs.length) { + lines.push( + '\\* differs by scenario (cell shows the worst):', + '', + ...differs, + '' + ); + } + lines.push('', ''); + } + + // Unexcused failures are already listed under Regressions; this table is + // the baselined remainder, for context. + const failures = []; + for (const s of sdks) { + const m = s.modes[mode]; + for (const sc of scenarioNames) { + for (const c of judgeScenario(m, sc).baselined) { + failures.push( + `| ${code(s.spec)} | ${code(sc)} | ${ICON.BASELINED} ${code(c.id)} | ${cell(c.errorMessage ?? c.description ?? '', 200) || DASH} |` + ); + } + } + } + if (failures.length) { + const cap = 80; + lines.push( + `
${mode} baselined failure messages (${failures.length})`, + '', + '| SDK | Scenario | Check | Message |', + '| --- | --- | --- | --- |', + ...failures.slice(0, cap) + ); + if (failures.length > cap) { + lines.push(`| | | | ${failures.length - cap} more in matrix.json |`); + } + lines.push('', '
', ''); + } + } + + const errs = []; + for (const s of sdks) { + for (const [mode, m] of Object.entries(s.modes)) { + if (m.error && !m.error.inherited) errs.push([s, mode, m.error]); + } + } + if (errs.length) { + lines.push('### Build and execution errors', ''); + for (const [s, mode, e] of errs) { + const tail = tailLines(String(e.tail ?? ''), 25).replace(/```/g, "'''"); + lines.push( + `
${cell(s.spec)} (${mode}): ${cell(errorLabel(e), 160)}`, + '', + `${cell(e.reason ?? '', 300)} (log: ${code(e.logFile ?? 'n/a')})`, + '', + '```text', + tail || '(no output captured)', + '```', + '', + '
', + '' + ); + } + } + return `${lines.join('\n').trimEnd()}\n`; +} + +export function hasErrors(matrix) { + return Object.values(matrix.sdks).some((s) => + Object.values(s.modes).some( + (m) => m.error || Object.values(m.scenarios).some((sc) => sc.missing) + ) + ); +} + +function exitCodeFor(opts, matrix) { + if (opts.strict && hasRed(matrix)) return 1; + if (opts.strictErrors && hasErrors(matrix)) return 1; + return 0; +} + +/** + * True when anything in the matrix would turn some SDK's own CI red, or could + * not be determined: an unexcused failure, a stale baseline entry, or an SDK + * that could not be built or run. + */ +export function hasRed(matrix) { + const reg = findRegressions(matrix); + return ( + reg.unexpected.length > 0 || reg.stale.length > 0 || reg.errors.length > 0 + ); +} + +function writeOutputs(matrix, outDir) { + fs.mkdirSync(outDir, { recursive: true }); + const md = renderMarkdown(matrix); + fs.writeFileSync( + path.join(outDir, 'matrix.json'), + JSON.stringify(matrix, null, 2) + ); + fs.writeFileSync(path.join(outDir, 'matrix.md'), md); + process.stdout.write(md); + console.error( + `\n[matrix] Wrote ${path.join(outDir, 'matrix.json')} and matrix.md` + ); +} + +// --------------------------------------------------------------------------- +// main + +export async function main(argv) { + let opts; + try { + opts = parseArgs(argv); + } catch (err) { + console.error(`${err.message}\n\n${HELP}`); + return 2; + } + if (opts.help) { + process.stdout.write(HELP); + return 0; + } + const outDir = path.resolve(opts.output); + if (!opts.listSdks) fs.mkdirSync(outDir, { recursive: true }); + const cacheDir = path.resolve( + opts.cacheDir ?? path.join(REPO_ROOT, '.sdk-under-test') + ); + + if (opts.merge.length) { + const files = findMatrixFiles(opts.merge); + if (files.length === 0) { + console.error( + `[matrix] No matrix.json found under: ${opts.merge.join(', ')}` + ); + return 1; + } + console.error(`[matrix] Merging ${files.length} matrix.json file(s)`); + const merged = mergeMatrices( + files.map((f) => JSON.parse(fs.readFileSync(f, 'utf-8'))) + ); + if (opts.title) merged.title = opts.title; + writeOutputs(merged, outDir); + return exitCodeFor(opts, merged); + } + + let harness; + try { + harness = await prepareHarness( + opts.listSdks ? { ...opts, skipHarnessBuild: true } : opts, + cacheDir, + outDir + ); + } catch (err) { + console.error(`[matrix] ${err.message}`); + return 1; + } + const known = listKnownSdks(harness.root); + if (opts.listSdks) { + process.stdout.write( + opts.json ? `${JSON.stringify(known)}\n` : `${known.join('\n')}\n` + ); + return 0; + } + const sdks = resolveSdkList(opts.sdks, known); + for (const s of sdks) { + if (!known.includes(sdkKey(s.name))) { + console.error( + `[matrix] warning: ${s.name} is not in KNOWN_SDKS at ${harness.ref} (known: ${known.join(', ')})` + ); + } + } + console.error( + `[matrix] Harness ${harness.ref} (${harness.sha}); SDKs: ${sdks.map((s) => s.spec).join(', ')}; mode ${opts.mode}; cache ${cacheDir}` + ); + + const ctx = { opts, harness, cacheDir, outDir, serverLock: createLock() }; + const records = await pool(sdks, opts.concurrency, async (sdk) => { + try { + return await runSdk(sdk, ctx); + } catch (err) { + // An orchestration bug for one SDK is recorded, never fatal. + const mode = opts.mode === 'both' ? 'client' : opts.mode; + return { + spec: sdk.spec, + name: sdk.name, + requestedRef: sdk.ref ?? null, + head: null, + checkoutDir: null, + toolchain: {}, + modes: { + [mode]: { + invocations: [], + error: { + phase: 'setup', + reason: `sdk-matrix internal error: ${err.message}`, + detail: null, + logFile: null, + tail: String(err.stack ?? err) + }, + scenarios: {} + } + } + }; + } + }); + + const matrix = { + title: opts.title ?? `SDK matrix: conformance ${harness.ref}`, + generatedAt: new Date().toISOString(), + harness: { ref: harness.ref, sha: harness.sha, version: harness.version }, + selection: { + mode: opts.mode, + scenario: opts.scenario ?? null, + suite: opts.suite ?? null, + requirements: opts.requirements ?? null, + specVersion: opts.specVersion ?? null + }, + host: { + platform: process.platform, + arch: process.arch, + node: process.version, + runner: process.env.GITHUB_ACTIONS + ? 'github-actions' + : process.env.SDK_MATRIX_IN_DOCKER + ? 'docker' + : 'local' + }, + sdks: Object.fromEntries(records.map((r) => [r.spec, r])) + }; + writeOutputs(matrix, outDir); + return exitCodeFor(opts, matrix); +} + +const invokedDirectly = + process.argv[1] && + import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href; +if (invokedDirectly) { + main(process.argv.slice(2)).then( + (code) => process.exit(code), + (err) => { + console.error(err); + process.exit(1); + } + ); +} diff --git a/scripts/sdk-matrix.test.ts b/scripts/sdk-matrix.test.ts new file mode 100644 index 00000000..19db02df --- /dev/null +++ b/scripts/sdk-matrix.test.ts @@ -0,0 +1,526 @@ +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { createRequire } from 'node:module'; +import { KNOWN_SDKS } from '../src/sdk-runner/known-sdks'; +import { + REPO_ROOT, + cell, + classifyInvocation, + collectModeResults, + findMatrixFiles, + hasRed, + judgeScenario, + listKnownSdks, + mergeMatrices, + parseArgs, + parseBaselineYaml, + parseSdkSpec, + renderMarkdown, + resolveSdkList, + summarizeChecks, + worstStatus + // @ts-expect-error untyped .mjs script +} from './sdk-matrix.mjs'; + +// Canned result dirs mirror what `conformance client|server -o ` writes: +// /-/checks.json, nested when the scenario name has a +// '/', and prefixed `server-` in server mode. +let tmp: string; + +function writeChecks(rel: string, checks: unknown) { + const dir = path.join(tmp, rel); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'checks.json'), JSON.stringify(checks)); +} + +const check = (id: string, status: string, extra = {}) => ({ + id, + name: id, + description: `desc of ${id}`, + status, + timestamp: '2026-09-06T00:00:00.000Z', + ...extra +}); + +beforeAll(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'sdk-matrix-test-')); + // ts-sdk client: auth/metadata-default ran twice (older run failed, newer + // passed) plus initialize. + writeChecks('ts/client/auth/metadata-default-2026-09-06T10-00-00-000Z', [ + check('prm-discovery', 'FAILURE', { errorMessage: 'old run' }) + ]); + writeChecks('ts/client/auth/metadata-default-2026-09-06T11-00-00-000Z', [ + check('prm-discovery', 'SUCCESS'), + check('resource-parameter-matches-prm', 'SUCCESS'), + check('token-request', 'SUCCESS'), + check('token-request', 'SUCCESS') + ]); + // Same check id with a different outcome in a sibling scenario, to + // exercise the "differs by scenario" breakout of the combined check table. + writeChecks('ts/client/auth/metadata-var2-2026-09-06T11-00-02-000Z', [ + check('prm-discovery', 'FAILURE', { errorMessage: 'var2 only' }) + ]); + writeChecks('ts/client/initialize-2026-09-06T11-00-01-000Z', [ + check('mcp-client-initialization', 'SUCCESS'), + check('server-info', 'INFO') + ]); + // go-sdk client: one warning, one failure with a message containing + // markdown-hostile characters. + writeChecks('go/client/auth/metadata-default-2026-09-06T11-00-00-000Z', [ + check('prm-discovery', 'SUCCESS'), + check('resource-parameter-matches-prm', 'FAILURE', { + errorMessage: + 'resource=http://a|b @octocat\nsecond line' + }), + check('token-request', 'WARNING', { errorMessage: 'slow' }) + ]); + // server mode naming + writeChecks('go/server/server-tools-list-2026-09-06T12-00-00-000Z', [ + check('tools-list', 'SUCCESS') + ]); + writeChecks('go/server/server-server-initialize-2026-09-06T12-00-00-000Z', [ + check('server-initialize', 'SUCCESS') + ]); +}); + +afterAll(() => { + fs.rmSync(tmp, { recursive: true, force: true }); +}); + +describe('parseArgs', () => { + it('defaults to all SDKs, client mode', () => { + const o = parseArgs([]); + expect(o.sdks).toBe('all'); + expect(o.mode).toBe('client'); + expect(o.concurrency).toBe(2); + }); + + it('accepts --key=value and repeated --merge', () => { + const o = parseArgs([ + '--mode=server', + '--merge', + 'a,b', + '--merge=c', + '--strict' + ]); + expect(o.mode).toBe('server'); + expect(o.merge).toEqual(['a', 'b', 'c']); + expect(o.strict).toBe(true); + }); + + it('treats an empty value as not given (unset workflow inputs)', () => { + const o = parseArgs(['--scenario', '', '--suite', 'auth', '--ref', '']); + expect(o.scenario).toBeUndefined(); + expect(o.suite).toBe('auth'); + expect(o.ref).toBeUndefined(); + }); + + it('rejects conflicting selections and bad modes', () => { + expect(() => parseArgs(['--scenario', 'a', '--suite', 'b'])).toThrow(); + expect(() => parseArgs(['--mode', 'sideways'])).toThrow(); + expect(() => parseArgs(['--bogus'])).toThrow(/Unknown argument/); + }); +}); + +describe('KNOWN_SDKS discovery', () => { + it('parses the same keys the module exports', () => { + expect(listKnownSdks(REPO_ROOT)).toEqual(Object.keys(KNOWN_SDKS)); + }); + + it('resolves all / explicit lists with refs', () => { + const known = ['typescript-sdk', 'go-sdk']; + expect(resolveSdkList('all', known).map((s: any) => s.spec)).toEqual(known); + expect(resolveSdkList('go-sdk@v1.2.0, someone/rust-sdk', known)).toEqual([ + { spec: 'go-sdk@v1.2.0', name: 'go-sdk', ref: 'v1.2.0' }, + { spec: 'someone/rust-sdk', name: 'someone/rust-sdk', ref: undefined } + ]); + expect(parseSdkSpec('typescript-sdk@')).toEqual({ + spec: 'typescript-sdk@', + name: 'typescript-sdk', + ref: undefined + }); + }); +}); + +describe('classifyInvocation', () => { + it('recognises a run that reached the scenarios', () => { + const out = [ + '[sdk] Fetching go-sdk (cached at /x)', + '[sdk] Building: go build ./...', + '', + '[sdk] conformance client --command ./c --scenario initialize', + 'Passed: 1/1' + ].join('\n'); + expect(classifyInvocation(out, 1)).toEqual({ phase: 'ran', exitCode: 1 }); + }); + + it('classifies a build failure and surfaces the toolchain error', () => { + const out = [ + '[sdk] Cloning https://github.com/modelcontextprotocol/rust-sdk.git -> /c/rust-sdk/main', + '[sdk] HEAD is 3023198', + '[sdk] Building: cargo build -p mcp-conformance', + ' Updating index', + 'error: failed to select a version for the requirement `process-wrap = "^10.0"`', + '[sdk] Command failed (exit 101): cargo build -p mcp-conformance' + ].join('\n'); + const c = classifyInvocation(out, 1); + expect(c.phase).toBe('build'); + expect(c.reason).toBe( + 'Command failed (exit 101): cargo build -p mcp-conformance' + ); + expect(c.detail).toMatch(/process-wrap/); + }); + + it('classifies a missing toolchain as a build failure', () => { + const out = [ + '[sdk] Fetching ruby-sdk (cached at /c/ruby-sdk/main)', + '[sdk] Building: bundle install', + '/bin/sh: 1: bundle: not found', + '[sdk] Command failed (exit 127): bundle install' + ].join('\n'); + const c = classifyInvocation(out, 1); + expect(c.phase).toBe('build'); + expect(c.reason).toMatch(/exit 127/); + }); + + it('classifies a server that never became ready', () => { + const out = [ + '[sdk] Fetching go-sdk (cached at /x)', + '[sdk] Building: go build', + '[sdk] Starting server: ./server', + '[sdk] Stopping server', + '[sdk] Server at http://localhost:3000 did not become ready within 15000ms: fetch failed' + ].join('\n'); + const c = classifyInvocation(out, 1); + expect(c.phase).toBe('server-start'); + expect(c.reason).toMatch(/did not become ready/); + }); + + it('classifies a checkout failure', () => { + const out = + "[sdk] Cloning https://github.com/x/y.git -> /c\n[sdk] Ref 'nope' not found in y (tried origin/nope, nope)"; + expect(classifyInvocation(out, 1).phase).toBe('checkout'); + }); +}); + +describe('collectModeResults', () => { + it('keys by scenario name, strips timestamps, keeps the latest run', () => { + const r = collectModeResults(path.join(tmp, 'ts/client'), 'client'); + expect(Object.keys(r).sort()).toEqual([ + 'auth/metadata-default', + 'auth/metadata-var2', + 'initialize' + ]); + expect(r['auth/metadata-default'].checks.map((c: any) => c.status)).toEqual( + ['SUCCESS', 'SUCCESS', 'SUCCESS', 'SUCCESS'] + ); + }); + + it('strips exactly one server- prefix in server mode', () => { + const r = collectModeResults(path.join(tmp, 'go/server'), 'server'); + expect(Object.keys(r).sort()).toEqual(['server-initialize', 'tools-list']); + }); + + it('returns {} for a missing dir', () => { + expect(collectModeResults(path.join(tmp, 'nope'), 'client')).toEqual({}); + }); +}); + +describe('aggregation helpers', () => { + it('worstStatus ranks FAILURE > WARNING > SUCCESS > INFO > SKIPPED', () => { + expect(worstStatus(['SUCCESS', 'INFO'])).toBe('SUCCESS'); + expect(worstStatus(['SUCCESS', 'WARNING', 'SKIPPED'])).toBe('WARNING'); + expect(worstStatus(['WARNING', 'FAILURE'])).toBe('FAILURE'); + }); + + it('summarizeChecks scores SUCCESS+FAILURE only', () => { + expect( + summarizeChecks([ + { status: 'SUCCESS' }, + { status: 'FAILURE' }, + { status: 'WARNING' }, + { status: 'INFO' } + ]) + ).toEqual({ passed: 1, failed: 1, warnings: 1, total: 2, emitted: 4 }); + }); + + it('cell() neutralises markdown/HTML/mentions and bounds length', () => { + const c = cell('a|b @octocat\nnext `tick` [l](http://x)', 200); + expect(c).not.toMatch(/[<>\n]/); + expect(c).toContain('a\\|b'); + expect(c).toContain('<img'); + expect(c).toContain('@​octocat'); + expect(c).not.toContain('`'); + expect(c).toContain('\\[l\\](http://x)'); + expect(cell('x'.repeat(500), 20)).toHaveLength(20); + }); +}); + +function matrixFrom(sdks: Record) { + return { + title: 'SDK matrix: conformance PR #488', + generatedAt: '2026-09-06T12:00:00.000Z', + harness: { ref: 'PR #488', sha: 'abc123def456', version: '0.2.0' }, + selection: { + mode: 'client', + scenario: 'auth/metadata-default', + suite: null, + requirements: null + }, + host: { runner: 'local' }, + sdks + }; +} + +describe('renderMarkdown', () => { + it('renders SDK x check cells, build-failed cells, and escapes data', () => { + const ts = { + spec: 'typescript-sdk', + name: 'typescript-sdk', + head: 'aaaaaaa', + toolchain: { node: 'v22.1.0', pnpm: '10.26.1', npm: '10.9.0' }, + modes: { + client: { + invocations: [], + error: null, + // var2's failure is excused wholesale; 'initialize' passes, so that + // entry is stale. + baseline: { + file: 'test/conformance/expected-failures.yaml', + entries: ['auth/metadata-var2', 'initialize'] + }, + scenarios: collectModeResults(path.join(tmp, 'ts/client'), 'client') + } + } + }; + const go = { + spec: 'go-sdk', + name: 'go-sdk', + head: 'bbbbbbb', + toolchain: { go: 'go version go1.26.5 linux/amd64' }, + modes: { + client: { + invocations: [], + error: null, + // Only the WARNING is excused (per-check); the FAILURE is not. + baseline: { + file: 'conformance/baseline.yml', + entries: ['auth/metadata-default:token-request'] + }, + scenarios: collectModeResults(path.join(tmp, 'go/client'), 'client') + } + } + }; + const rust = { + spec: 'rust-sdk', + name: 'rust-sdk', + head: 'ccccccc', + toolchain: { cargo: 'cargo 1.96.1 (abc 2026-01-01)', rustc: null }, + modes: { + client: { + invocations: [], + error: { + phase: 'build', + reason: 'Command failed (exit 101): cargo build -p mcp-conformance', + detail: + 'failed to select a version for the requirement `process-wrap = "^10.0"`', + logFile: 'sdks/rust-sdk/client-auth_metadata-default.log', + tail: 'error: failed to select a version\n```injected fence```' + }, + scenarios: { + 'auth/metadata-default': { + resultDir: null, + checks: [], + missing: true, + reason: 'x' + } + } + } + } + }; + const md: string = renderMarkdown( + matrixFrom({ 'typescript-sdk': ts, 'go-sdk': go, 'rust-sdk': rust }) + ); + + // Overview rows. INFO is not scored: ts has 5 SUCCESS + 1 FAILURE, and + // that failure is baselined while 'initialize' is a stale entry. + expect(md).toContain( + '| `typescript-sdk` | `aaaaaaa` | ⚠️ 1 stale baseline, 1 baselined; 5/6 checks pass | `test/conformance/expected-failures.yaml` |' + ); + expect(md).toMatch( + /\| `go-sdk` \| `bbbbbbb` \| ❌ 1 unexpected, 1 baselined; 1\/2 checks pass \| `conformance\/baseline.yml` \| go 1\.26\.5 \|/ + ); + expect(md).toContain( + "❌ build failed: failed to select a version for the requirement 'process-wrap" + ); + expect(md).toContain('rustc missing'); + + // Regressions section: the one unexcused failure, the stale entry, and + // the SDK that could not run. + const regressions = md.slice( + md.indexOf('### Regressions'), + md.indexOf('### client') + ); + expect(regressions).toContain( + "1 failing check(s) not covered by the SDK's baseline" + ); + expect(regressions).toMatch( + /\| `go-sdk` \| client \| `auth\/metadata-default` \| ❌ `resource-parameter-matches-prm` \|/ + ); + expect(regressions).not.toContain('token-request'); + expect(regressions).not.toContain('prm-discovery'); + expect(regressions).toContain('- `typescript-sdk` client: `initialize`'); + expect(regressions).toMatch(/- `rust-sdk` client: build failed/); + + // Scenario summary cells distinguish unexcused / baselined / stale. + expect(md).toMatch( + /\| `initialize` \| ✅ 1\/1 🧹stale baseline \| — \| ❌ build failed/ + ); + expect(md).toMatch( + /\| `auth\/metadata-default` \| ✅ 4\/4 \| ❌ 1\/2 \(\+1⚠️\) \|/ + ); + expect(md).toMatch( + /\| `auth\/metadata-var2` \| ⭕ 0\/1 baselined \| — \| ❌ build failed/ + ); + // One combined SDK x check table per mode: ids are unioned across the + // three scenarios, a repeated id collapses to one row, and a baselined + // failure renders as ⭕ rather than ❌/⚠️. + expect(md).toContain('client checks: 5 (3 scenarios)'); + expect(md.match(/^\| Check \|/gm)?.length).toBe(1); + const rows = md + .split('\n') + .filter((l) => l.startsWith('| `token-request` |')); + expect(rows).toEqual(['| `token-request` | ✅ | ⭕ | — |']); + expect(md).toContain('| `resource-parameter-matches-prm` | ✅ | ❌ | — |'); + expect(md).toContain('| `mcp-client-initialization` | ✅ | — | — |'); + // A check whose outcome differs between scenarios is starred and listed. + expect(md).toContain('| `prm-discovery` | ⭕\\* | ✅ | — |'); + expect(md).toContain( + '- `typescript-sdk` `prm-discovery`: ✅ `auth/metadata-default`, ⭕ `auth/metadata-var2`' + ); + // Baselined failures keep their messages in the per-mode table. + expect(md).toContain( + 'client baselined failure messages (2)' + ); + // Failure messages are escaped data. + expect(md).not.toContain('