diff --git a/.github/workflows/lint-build-test.yml b/.github/workflows/lint-build-test.yml index 36039484255..daa366987da 100644 --- a/.github/workflows/lint-build-test.yml +++ b/.github/workflows/lint-build-test.yml @@ -3,6 +3,9 @@ name: Lint, Build, and Test on: workflow_call: +permissions: + contents: read + jobs: prepare: name: Prepare @@ -12,6 +15,9 @@ jobs: node-version: [18.x, 20.x, 22.x, 24.x] outputs: child-workspace-package-names: ${{ steps.workspace-package-names.outputs.child-workspace-package-names }} + merge-base: ${{ steps.fetch-merge-base.outputs.merge-base }} + package-names: ${{ steps.packages.outputs.package-names }} + changed-paths: ${{ steps.packages.outputs.changed-paths }} steps: - name: Checkout and setup environment uses: MetaMask/action-checkout-and-setup@v3 @@ -26,6 +32,46 @@ jobs: run: | echo "child-workspace-package-names=$(yarn workspaces list --no-private --json | jq --slurp --raw-output 'map(.name) | @json')" >> "$GITHUB_OUTPUT" shell: bash + - name: Fetch merge base + id: fetch-merge-base + if: matrix.node-version == '24.x' && (github.base_ref != '' || github.event.merge_group.base_ref != '') + run: | + set -euo pipefail + + PREFIXED_REF_REGEX='refs/heads/(.+)' + if [[ "$BASE_REF" =~ $PREFIXED_REF_REGEX ]]; then + BASE_REF="${BASH_REMATCH[1]}" + fi + + MERGE_BASE=$(gh api "repos/$GITHUB_REPOSITORY/compare/$BASE_REF...$HEAD_SHA" --jq '.merge_base_commit.sha') + git fetch --unshallow --filter=blob:none --no-tags origin HEAD + + echo "merge-base=$MERGE_BASE" >> "$GITHUB_OUTPUT" + env: + BASE_REF: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + GH_TOKEN: ${{ github.token }} + - name: Get changed package names + id: packages + if: matrix.node-version == '24.x' + run: | + if [[ -n "$MERGE_BASE" ]]; then + OUTPUT=$(yarn tsx scripts/get-changed-workspaces.mts "$MERGE_BASE" "$HEAD_SHA") + PACKAGES=$(echo "$OUTPUT" | jq -c '.names') + if [[ $(echo "$OUTPUT" | jq '.hasRootChange') == "true" ]]; then + CHANGED_PATHS="full" + else + CHANGED_PATHS=$(echo "$OUTPUT" | jq -c '.locations') + fi + else + PACKAGES=$(yarn workspaces list --no-private --json | jq --slurp --raw-output 'map(.name) | @json') + CHANGED_PATHS="full" + fi + echo "package-names=$PACKAGES" >> "$GITHUB_OUTPUT" + echo "changed-paths=$CHANGED_PATHS" >> "$GITHUB_OUTPUT" + env: + MERGE_BASE: ${{ steps.fetch-merge-base.outputs.merge-base }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} lint: name: Lint (${{ matrix.script }}) @@ -35,7 +81,6 @@ jobs: matrix: node-version: [24.x] script: - - lint:eslint - lint:misc:check - constraints - lint:dependencies @@ -61,14 +106,52 @@ jobs: exit 1 fi + lint-eslint: + name: Lint (lint:eslint) + runs-on: ubuntu-latest + if: needs.prepare.outputs.changed-paths != '[]' + needs: prepare + strategy: + matrix: + node-version: [24.x] + steps: + - name: Checkout and setup environment + uses: MetaMask/action-checkout-and-setup@v3 + with: + is-high-risk-environment: false + persist-credentials: false + node-version: ${{ matrix.node-version }} + - name: Lint + run: | + if [[ "$CHANGED_PATHS" == "full" ]]; then + echo "Running ESLint on all packages." + echo "" + yarn lint:eslint + else + echo "Running ESLint on:" + echo "$CHANGED_PATHS" | jq -r '"- " + .[]' + echo "" + echo "$CHANGED_PATHS" | jq -r '.[]' | xargs yarn lint:eslint + fi + env: + CHANGED_PATHS: ${{ needs.prepare.outputs.changed-paths }} + - name: Require clean working directory + shell: bash + run: | + if ! git diff --exit-code; then + echo "Working tree dirty at end of job" + exit 1 + fi + validate-changelog: name: Validate changelog runs-on: ubuntu-latest + if: needs.prepare.outputs.package-names != '[]' needs: prepare strategy: matrix: node-version: [24.x] - package-name: ${{ fromJson(needs.prepare.outputs.child-workspace-package-names) }} + package-name: ${{ fromJson(needs.prepare.outputs.package-names) }} steps: - name: Checkout and setup environment uses: MetaMask/action-checkout-and-setup@v3 @@ -113,7 +196,36 @@ jobs: is-high-risk-environment: false persist-credentials: false node-version: ${{ matrix.node-version }} - - run: yarn build + - name: Unshallow checkout + if: needs.prepare.outputs.merge-base != '' + run: | + # Unshallow so git can walk history back to the merge base for + # `git diff --name-only`. Using `--filter=blob:none` avoids + # downloading file content — only commit and tree objects are needed. + git fetch --unshallow --filter=blob:none --no-tags origin HEAD + - name: Build + run: | + if [[ -n "$MERGE_BASE" && "$CHANGED_PATHS" != "full" ]]; then + TSCONFIG=$(mktemp --tmpdir="$GITHUB_WORKSPACE" --suffix=.json) + yarn tsx scripts/generate-partial-build-tsconfig.mts "$MERGE_BASE" "$HEAD_SHA" > "$TSCONFIG" + if [[ -s "$TSCONFIG" ]]; then + echo "Building changed packages:" + jq -r '"- " + (.references[].path | ltrimstr("./") | rtrimstr("/tsconfig.build.json"))' "$TSCONFIG" + echo "" + yarn ts-bridge --project "$TSCONFIG" --verbose + else + echo "No packages to build." + fi + rm -f "$TSCONFIG" + else + echo "Building all packages." + echo "" + yarn build + fi + env: + MERGE_BASE: ${{ needs.prepare.outputs.merge-base }} + HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + CHANGED_PATHS: ${{ needs.prepare.outputs.changed-paths }} - name: Require clean working directory shell: bash run: | @@ -151,10 +263,11 @@ jobs: test-18: name: Test (18.x) runs-on: ubuntu-latest + if: needs.prepare.outputs.package-names != '[]' needs: prepare strategy: matrix: - package-name: ${{ fromJson(needs.prepare.outputs.child-workspace-package-names) }} + package-name: ${{ fromJson(needs.prepare.outputs.package-names) }} steps: - name: Checkout and setup environment uses: MetaMask/action-checkout-and-setup@v3 @@ -176,10 +289,11 @@ jobs: test-20: name: Test (20.x) runs-on: ubuntu-latest + if: needs.prepare.outputs.package-names != '[]' needs: prepare strategy: matrix: - package-name: ${{ fromJson(needs.prepare.outputs.child-workspace-package-names) }} + package-name: ${{ fromJson(needs.prepare.outputs.package-names) }} steps: - name: Checkout and setup environment uses: MetaMask/action-checkout-and-setup@v3 @@ -201,10 +315,11 @@ jobs: test-22: name: Test (22.x) runs-on: ubuntu-latest + if: needs.prepare.outputs.package-names != '[]' needs: prepare strategy: matrix: - package-name: ${{ fromJson(needs.prepare.outputs.child-workspace-package-names) }} + package-name: ${{ fromJson(needs.prepare.outputs.package-names) }} steps: - name: Checkout and setup environment uses: MetaMask/action-checkout-and-setup@v3 @@ -246,6 +361,7 @@ jobs: test-wallet-cli-e2e: name: Test wallet-cli daemon e2e (${{ matrix.node-version }}) runs-on: ubuntu-latest + if: needs.prepare.outputs.package-names == '[]' || contains(fromJson(needs.prepare.outputs.package-names), '@metamask/wallet-cli') needs: prepare strategy: matrix: diff --git a/scripts/generate-partial-build-tsconfig.mts b/scripts/generate-partial-build-tsconfig.mts new file mode 100644 index 00000000000..1df7600fb8b --- /dev/null +++ b/scripts/generate-partial-build-tsconfig.mts @@ -0,0 +1,52 @@ +import { + getAllWorkspaces, + getTypeScriptWorkspaces, + computeChangedWorkspaces, +} from './lib/workspaces.mjs'; + +/** + * Generate a filtered tsconfig.build.json for partial CI builds. + * + * Given a merge base SHA, outputs a tsconfig that references only the + * TypeScript packages that changed since that commit plus their transitive + * dependants and dependencies. Pipe the output to a temp file and pass it + * to `ts-bridge --project`. + * + * Dependencies are always included because TypeScript project references + * require every referenced project's dist output to already exist on disk. + * + * Usage: `tsx scripts/generate-partial-build-tsconfig.mts []` + */ +async function main() { + const mergeBase = process.argv[2]; + if (!mergeBase) { + console.error( + 'Usage: generate-partial-build-tsconfig.mts []', + ); + process.exitCode = 1; + return; + } + + const headRef = process.argv[3] ?? 'HEAD'; + + const allWorkspaces = await getAllWorkspaces(); + const typeScriptWorkspaces = await getTypeScriptWorkspaces(); + const packagesToBuild = await computeChangedWorkspaces( + allWorkspaces, + mergeBase, + headRef, + true, + ); + + const references = typeScriptWorkspaces + .filter(({ name }) => packagesToBuild.has(name)) + .map(({ location }) => ({ path: `./${location}/tsconfig.build.json` })); + + if (references.length === 0) { + return; + } + + console.log(JSON.stringify({ files: [], include: [], references }, null, 2)); +} + +await main(); diff --git a/scripts/get-changed-workspaces.mts b/scripts/get-changed-workspaces.mts new file mode 100644 index 00000000000..03d1528b35b --- /dev/null +++ b/scripts/get-changed-workspaces.mts @@ -0,0 +1,64 @@ +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; + +import { + getAllWorkspaces, + checkRootChange, + computeChangedWorkspaces, + getChangedFiles, +} from './lib/workspaces.mjs'; + +/** + * List workspaces that need to be checked given a merge base. + * + * Outputs a JSON object to stdout: + * - `names`: package names of changed workspaces and their transitive dependants + * - `locations`: workspace-relative paths (e.g. `packages/foo`) for the same set + * - `hasRootChange`: true if any non-ignored root file changed (triggers a full run) + * + * Usage: `tsx scripts/get-changed-workspaces.mts [] [options]` + */ +const argv = await yargs(hideBin(process.argv)) + .usage('$0 [head-sha] [options]') + .positional('merge-base', { + type: 'string', + describe: 'Merge base SHA', + }) + .positional('head-sha', { + type: 'string', + describe: 'PR branch tip SHA (defaults to HEAD)', + }) + .option('include-dependencies', { + type: 'boolean', + default: false, + describe: + 'Also expand to transitive dependencies (needed for TypeScript builds)', + }) + .demandCommand(1, 'merge-base is required') + .help() + .parseAsync(); + +const mergeBase = argv._[0] as string; +const headRef = (argv._[1] as string | undefined) ?? 'HEAD'; +const workspaces = await getAllWorkspaces(); + +const changedFiles = await getChangedFiles(mergeBase, headRef); +const hasRootChange = checkRootChange(workspaces, changedFiles); + +const changed = await computeChangedWorkspaces( + workspaces, + mergeBase, + headRef, + argv['include-dependencies'], + changedFiles, +); + +const changedWorkspaces = workspaces.filter(({ name }) => changed.has(name)); + +console.log( + JSON.stringify({ + names: changedWorkspaces.map(({ name }) => name), + locations: changedWorkspaces.map(({ location }) => location), + hasRootChange, + }), +); diff --git a/scripts/lib/workspaces.mts b/scripts/lib/workspaces.mts new file mode 100644 index 00000000000..fbfec573cf9 --- /dev/null +++ b/scripts/lib/workspaces.mts @@ -0,0 +1,212 @@ +import { fileExists } from '@metamask/utils/node'; +import execa from 'execa'; +import { readFile } from 'node:fs/promises'; +import { join } from 'node:path'; + +export const ROOT_WORKSPACE = new URL('../..', import.meta.url).pathname; + +// Files that can change without requiring a full rebuild/test run. +const IGNORED_ROOT_FILES = new Set([ + '.gitignore', + 'AGENTS.md', + 'CLAUDE.md', + 'README.md', + 'teams.json', + 'yarn.lock', +]); + +export type Workspace = { + location: string; + name: string; +}; + +export type DependencyGraph = { + dependants: Record>; + dependencies: Record>; +}; + +/** + * Get all non-root workspaces in the monorepo. + * + * @returns All workspaces. + */ +export async function getAllWorkspaces(): Promise { + const { stdout } = await execa( + 'yarn', + ['workspaces', 'list', '--no-private', '--json'], + { + cwd: ROOT_WORKSPACE, + encoding: 'utf8', + }, + ); + + return stdout + .trim() + .split('\n') + .map((line) => JSON.parse(line)) + .filter(({ location }: Workspace) => location !== '.'); +} + +/** + * Get all TypeScript workspaces in the monorepo. This filters to packages + * containing a "tsconfig.build.json" file. + * + * @returns All TypeScript workspaces. + */ +export async function getTypeScriptWorkspaces(): Promise { + const workspaces = await getAllWorkspaces(); + + return ( + await Promise.all( + workspaces.map(async (workspace) => { + const hasTsConfig = await fileExists( + join(ROOT_WORKSPACE, workspace.location, 'tsconfig.build.json'), + ); + return hasTsConfig ? workspace : null; + }), + ) + ).filter((workspace): workspace is Workspace => workspace !== null); +} + +/** + * Get dependency and dependant maps for all workspaces. + * + * @param workspaces - The workspaces to build the graph for. + * @returns Maps of package name to dependants and package name to dependencies. + */ +export async function getWorkspaceDependencies( + workspaces: Workspace[], +): Promise { + const dependants: Record> = Object.fromEntries( + workspaces.map(({ name }) => [name, new Set()]), + ); + const dependencies: Record> = Object.fromEntries( + workspaces.map(({ name }) => [name, new Set()]), + ); + + const packages = await Promise.all( + workspaces.map(({ location }) => + readFile(join(ROOT_WORKSPACE, location, 'package.json'), { + encoding: 'utf-8', + }).then(JSON.parse), + ), + ); + + for (const [index, { name }] of workspaces.entries()) { + const pkg = packages[index]; + for (const dependency of Object.keys({ + ...pkg.dependencies, + ...pkg.devDependencies, + })) { + if (dependants[dependency] !== undefined) { + dependants[dependency].add(name); + dependencies[name].add(dependency); + } + } + } + + return { dependants, dependencies }; +} + +/** + * Get the list of files changed between the merge base and the PR head. + * + * @param mergeBase - The merge base SHA. + * @param headRef - The PR branch tip SHA (or "HEAD" as fallback). + * @returns A list of changed file paths. + */ +export async function getChangedFiles( + mergeBase: string, + headRef: string, +): Promise { + const { stdout } = await execa( + 'git', + ['diff', '--name-only', `${mergeBase}...${headRef}`], + { + cwd: ROOT_WORKSPACE, + encoding: 'utf8', + }, + ); + + return stdout.trim().split('\n').filter(Boolean); +} + +/** + * Check whether any changed file lives outside all package directories and + * is not in the ignored root files list. When true, a full + * rebuild/test/lint run is required. + * + * @param workspaces - The workspace set to check against. + * @param changedFiles - The list of changed file paths. + * @returns Whether any non-ignored root file changed. + */ +export function checkRootChange( + workspaces: Workspace[], + changedFiles: string[], +): boolean { + return changedFiles.some( + (file) => + !IGNORED_ROOT_FILES.has(file) && + !workspaces.some(({ location }) => file.startsWith(`${location}/`)), + ); +} + +/** + * Compute the set of workspace names that need to be checked given a merge + * base, by finding changed packages and expanding to transitive dependants. + * + * When `includeDependencies` is true, also expands to transitive dependencies. + * This is needed for TypeScript project reference builds, where every + * referenced project's dist output must already exist on disk. + * + * @param workspaces - The workspace set to compute against. + * @param mergeBase - The merge base SHA. + * @param headRef - The PR branch tip SHA (or "HEAD" as fallback). + * @param includeDependencies - Whether to also expand to transitive dependencies. + * @param changedFiles - Pre-fetched list of changed files; fetched if omitted. + * @returns The set of workspace names to check. + */ +export async function computeChangedWorkspaces( + workspaces: Workspace[], + mergeBase: string, + headRef: string, + includeDependencies: boolean, + changedFiles?: string[], +): Promise> { + const files = changedFiles ?? (await getChangedFiles(mergeBase, headRef)); + const { dependants, dependencies } = + await getWorkspaceDependencies(workspaces); + + // If any changed file lives outside all package directories (e.g. root + // configs, workflow files, scripts), rebuild and test everything. + if (checkRootChange(workspaces, files)) { + return new Set(workspaces.map(({ name }) => name)); + } + + const result = new Set( + files.flatMap((file) => { + const workspace = workspaces.find(({ location }) => + file.startsWith(`${location}/`), + ); + return workspace ? [workspace.name] : []; + }), + ); + + // Expand to transitive dependants (packages that depend on what changed). + for (const pkg of result) { + for (const dependant of dependants[pkg] ?? []) { + result.add(dependant); + } + } + + if (includeDependencies) { + // Expand to transitive dependencies (dist files must exist to build dependants). + for (const pkg of result) { + for (const dependency of dependencies[pkg] ?? []) { + result.add(dependency); + } + } + } + + return result; +} diff --git a/tsconfig.scripts.json b/tsconfig.scripts.json index b4adc2c51e5..691fa67fa7e 100644 --- a/tsconfig.scripts.json +++ b/tsconfig.scripts.json @@ -18,6 +18,6 @@ "noErrorTruncation": true, "noUncheckedIndexedAccess": true }, - "include": ["./scripts/**/*.ts"], + "include": ["./scripts/**/*.ts", "./scripts/**/*.mts"], "exclude": ["**/node_modules"] }