build(webcam): stage ONNX Runtime so the segmentation effect can actually run

.github/workflows/ci.yml

@@ -159,9 +159,19 @@ jobs:
       # [env] block (cargo has no [target.<cfg>.env] — the macOS section in that file
       # is inert and cargo warns "unused key"), so before that change this job pointed
       # bindgen at the win64 tree and could never have gone green.
+      #
+      # Without a library on ORT_DYLIB_PATH, `runtime_available()` is false and every
+      # segmentation test returns early — the suite goes green having exercised no
+      # inference at all, which is exactly how the `ort`-panics-when-absent bug got
+      # in. Staging it here is what makes `the_whole_loop_produces_a_mask_from_
+      # compose_frame_alone` a real test on this runner instead of a skipped one.
+      # ~30 MB, next to nothing beside `brew install ffmpeg` above.
+      - name: Stage ONNX Runtime
+        run: node scripts/fetch-onnxruntime.mjs
       - name: cargo test (compositor, aarch64-apple-darwin)
         env:
           MAC_FFMPEG_DIR: /opt/homebrew/opt/ffmpeg
+          ORT_DYLIB_PATH: ${{ github.workspace }}/electron/native/bin/darwin-arm64/libonnxruntime.dylib
         run: |
           cd crates
           cargo test -p openscreen-compositor --lib --tests

THIRD-PARTY-NOTICES.md

@@ -49,6 +49,42 @@ distributed by their own registries, not redistributed inside our binaries.
 - The speech model (`ggml-*.bin`) is **not** bundled — it is downloaded into the
   user's data directory on first use by `electron/stt/modelManager.ts`.
 
+## ONNX Runtime (Windows and Apple Silicon macOS)
+
+- **Component**: `onnxruntime.dll` / `libonnxruntime.dylib`, under
+  `resources/electron/native/bin/<platform>-<arch>/`.
+- **License**: MIT — <https://github.com/microsoft/onnxruntime>.
+- Not built here: the pinned upstream release archive is downloaded, SHA-256
+  verified and unpacked by `scripts/fetch-onnxruntime.mjs`, which also checks the
+  archive's own LICENSE really is MIT before vendoring anything.
+- **Why it ships**: the native compositor segments the webcam subject with it, on
+  the CPU execution provider, to drive the camera background cutout/blur/custom
+  modes. The `gpu_cuda*` builds are deliberately not used — they are an order of
+  magnitude larger and carry NVIDIA redistribution terms.
+- **Not on Intel macOS**: upstream publishes no `osx-x86_64` asset from 1.27 on,
+  so the x64 DMG ships without it and the camera background effects are simply
+  absent there. Not shipped on Linux either, where the compositor has no capture
+  path for the mask yet.
+- The segmentation model it runs is a separate component, immediately below.
+
+## MediaPipe Selfie Segmentation — model weights
+
+- **Components**: `selfie_segmentation.tflite`,
+  `selfie_segmentation_landscape.tflite` and the `selfie_segmentation_landscape.onnx`
+  derived from them, shipped inside `app.asar` under `dist/mediapipe/`.
+- **License**: Apache-2.0 — <https://google.github.io/mediapipe/solutions/selfie_segmentation>.
+  Copyright The MediaPipe Authors.
+- The `.onnx` is a **derived work**, generated from the vendored `.tflite` by
+  `scripts/convert-selfie-segmentation-to-onnx.py`. No third-party weights are
+  downloaded at build time.
+- **Why it is listed here**: these weights are redistributed inside the installer,
+  and Apache-2.0 §4 asks that the attribution travel with them. The provenance note
+  in `public/mediapipe/selfie_segmentation/README.md` does not — electron-builder's
+  `"!*.md"` filter strips it from the package — so this file is the only copy a user
+  ever receives.
+- The MediaPipe **JavaScript** solution and its two ~5.6 MB WASM builds are no longer
+  bundled: inference moved into the native compositor, and nothing loaded them.
+
 ## Microsoft OpenMP runtime — `vcomp140.dll` (Windows only)
 
 - **Component**: `resources/electron/native/bin/win32-x64/vcomp140.dll`.

package.json

@@ -42,15 +42,15 @@
 		"assets:appx": "node scripts/generate-appx-assets.mjs",
 		"preview": "vite preview",
 		"build:native:mac": "node scripts/build-macos-screencapturekit-helper.mjs",
-		"build:mac": "npm run build:native:mac && npm run fetch:ffmpeg:mac && npm run build:native:compositor:mac && tsc && vite build && electron-builder --mac",
+		"build:mac": "npm run build:native:mac && npm run fetch:ffmpeg:mac && npm run fetch:onnxruntime && npm run build:native:compositor:mac && tsc && vite build && electron-builder --mac",
 		"build:native:win": "node scripts/build-windows-wgc-helper.mjs",
 		"stage:vcomp": "node scripts/stage-vcomp-runtime.mjs",
 		"build:native:compositor": "node scripts/build-windows-compositor-addon.mjs",
 		"build:native:compositor:mac": "node scripts/build-macos-compositor-addon.mjs",
 		"build:native:compositor:linux": "node scripts/build-linux-compositor-addon.mjs",
 		"build:native:linux": "node scripts/build-linux-pipewire-helper.mjs",
-		"build:win": "npm run build:native:win && npm run fetch:ffmpeg && npm run stage:vcomp && npm run build:native:compositor && tsc && vite build && electron-builder --win --config.npmRebuild=false",
-		"build:win:store": "npm run build:native:win && npm run fetch:ffmpeg && npm run stage:vcomp && npm run build:native:compositor && tsc && vite build && electron-builder --win appx --config.npmRebuild=false",
+		"build:win": "npm run build:native:win && npm run fetch:ffmpeg && npm run fetch:onnxruntime && npm run stage:vcomp && npm run build:native:compositor && tsc && vite build && electron-builder --win --config.npmRebuild=false",
+		"build:win:store": "npm run build:native:win && npm run fetch:ffmpeg && npm run fetch:onnxruntime && npm run stage:vcomp && npm run build:native:compositor && tsc && vite build && electron-builder --win appx --config.npmRebuild=false",
 		"build:linux": "npm run fetch:ffmpeg:sdk && npm run build:native:linux && npm run build:native:compositor:linux && tsc && vite build && electron-builder --linux AppImage deb pacman rpm --config.npmRebuild=false",
 		"build:whisper-binaries": "bash scripts/build-whisper-stt.sh",
 		"test:whisper-stt": "node scripts/test-whisper-stt.mjs",
@@ -92,7 +92,8 @@
 		"prepare": "husky",
 		"fetch:ffmpeg": "node scripts/fetch-ffmpeg.mjs",
 		"fetch:ffmpeg:mac": "node scripts/fetch-ffmpeg-macos.mjs",
-		"fetch:ffmpeg:sdk": "node scripts/fetch-ffmpeg.mjs --sdk-only"
+		"fetch:ffmpeg:sdk": "node scripts/fetch-ffmpeg.mjs --sdk-only",
+		"fetch:onnxruntime": "node scripts/fetch-onnxruntime.mjs"
 	},
 	"dependencies": {
 		"@fix-webm-duration/fix": "^1.0.1",

scripts/fetch-onnxruntime.mjs

@@ -0,0 +1,307 @@
+// Provisions the ONNX Runtime shared library into
+// electron/native/bin/<platform>-<arch>/, next to the compositor addon and the
+// ffmpeg libraries. That directory is gitignored and shipped by electron-builder's
+// extraResources, so this runs at build time rather than committing a 15-38 MB binary.
+//
+// WHY IT SHIPS: the native compositor segments the webcam subject on the CPU
+// execution provider (crates/compositor/src/segmentation.rs), and `ort` is linked
+// with `load-dynamic` — nothing is needed to BUILD, but at runtime
+// `ensureOnnxRuntimeOnPath` (electron/native-bridge/services/compositorViewService.ts)
+// walks this exact directory looking for the library and sets ORT_DYLIB_PATH to it.
+// Without it `Segmenter::load` fails, the compositor logs one line and draws the
+// webcam unsegmented — so the AI background cutout/blur/custom modes are simply off.
+// Everything degrades; nothing breaks. That is why this script never fails a build.
+//
+// VERSION IS NOT FREE TO MOVE. crates/Cargo.toml pins `ort` with feature `api-NN`,
+// which is the MINIMUM ONNX Runtime minor version the crate will accept — a lower
+// one makes `GetApi` return null and `ort` panics rather than erroring. The pin here
+// must satisfy that, and scripts/fetch-onnxruntime.test.mjs cross-checks the two so
+// a bump on either side cannot land alone.
+//
+// SUPPLY CHAIN. This binary is signed and shipped to every user, so nothing floats:
+//   - Pinned to an immutable release tag, never `latest`.
+//   - SHA-256 verified before the archive is opened. The digests below are the ones
+//     GitHub publishes per asset (`digest` in the releases API), independently
+//     re-verified by downloading and hashing.
+//   - Only the plain CPU assets. The `gpu_cuda*` variants are 200-320 MB and pull
+//     NVIDIA runtime dependencies we neither need nor may redistribute; the measured
+//     decision to use the CPU EP is in
+//     technical-documentation/engineering/webcam-segmentation.md.
+//
+// LICENSING: ONNX Runtime is MIT, which is compatible with this MIT app — but the
+// archive is checked rather than trusted, the same way fetch-ffmpeg.mjs verifies
+// ffmpeg's LGPL-ness instead of believing the asset name. Attribution ships in
+// THIRD-PARTY-NOTICES.md.
+
+import { spawnSync } from "node:child_process";
+import crypto from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const ROOT = path.join(__dirname, "..");
+
+/**
+ * The pinned release. Asset names are DERIVED from it rather than written out per
+ * entry, which is deliberate: fetch-ffmpeg.mjs keeps full asset strings and grew a
+ * test because a re-pin moved some and not others. Templating removes that failure
+ * mode by construction. The digests still have to move by hand — but a stale one
+ * fails loudly on the SHA-256 check before anything is extracted, which is the safe
+ * direction to fail in.
+ */
+const VERSION = "1.27.1";
+const BASE = `https://github.com/microsoft/onnxruntime/releases/download/v${VERSION}`;
+
+/**
+ * Per-target: the upstream artifact slug, its digest, and the library to lift out.
+ *
+ * `out` is not cosmetic — it is the exact name `ortLibName()` looks for in
+ * compositorViewService.ts. `member` is the file inside the archive, which on macOS
+ * and Linux is the VERSIONED real file rather than the unversioned symlink beside
+ * it: tar restores that symlink as a symlink, and a dangling one in the packaged app
+ * would resolve to nothing.
+ *
+ * darwin-x64 is absent and cannot be added: Microsoft publishes no `osx-x86_64`
+ * (or universal) asset for any release from 1.27 on — arm64 is the only macOS
+ * target. Building it from source is the only way to change that, and it is not
+ * worth an ffmpeg-macos-sized build script for a shrinking platform when the
+ * fallback is "the effect is off". See the darwin-x64 branch in main().
+ */
+const PINNED = {
+	"win32-x64": {
+		slug: "win-x64",
+		ext: "zip",
+		sha256: "2e00414a63fdef0914cd5a5ede6c707844878e0c08e1b6693842f0451b2df2a1",
+		member: "onnxruntime.dll",
+		out: "onnxruntime.dll",
+	},
+	"win32-arm64": {
+		slug: "win-arm64",
+		ext: "zip",
+		sha256: "6e22c2061ba6400b42a59663d700c8694e4e8fe654cf452c4700c24237407ae1",
+		member: "onnxruntime.dll",
+		out: "onnxruntime.dll",
+	},
+	"darwin-arm64": {
+		slug: "osx-arm64",
+		ext: "tgz",
+		sha256: "e42b77a7281cc6e55141bf44fcfbac2c782b823a491bbb6ac33c781dd991f8a6",
+		member: `libonnxruntime.${VERSION}.dylib`,
+		out: "libonnxruntime.dylib",
+	},
+	// Linux is pinned but not yet wired into `build:linux`: the Linux compositor
+	// back-end carries the segmentation SHADER only — nothing captures the webcam
+	// frame or uploads a mask there, so `fx.z` never leaves 0 and the library would
+	// be 23 MB of installer for a code path that cannot run. The entry is here so
+	// that landing the Linux capture half is a one-line change to package.json.
+	// See technical-documentation/engineering/webcam-segmentation-backend-port.md.
+	"linux-x64": {
+		slug: "linux-x64",
+		ext: "tgz",
+		sha256: "25b1ef1fea1acd210d63f8f24dc870ad6e077795ce1f54876252c6d3803c15af",
+		member: `libonnxruntime.so.${VERSION}`,
+		out: "libonnxruntime.so",
+	},
+	"linux-arm64": {
+		slug: "linux-aarch64",
+		ext: "tgz",
+		sha256: "33c67e33d1e25b816878366ea276589a024f71f000e7ff955c4b33224d639edd",
+		member: `libonnxruntime.so.${VERSION}`,
+		out: "libonnxruntime.so",
+	},
+};
+
+const assetName = (spec) => `onnxruntime-${spec.slug}-${VERSION}.${spec.ext}`;
+
+/** Magic bytes the vendored library must start with, per target platform. */
+const MAGIC = {
+	win32: { bytes: [0x4d, 0x5a], name: "PE (MZ)" }, //             .dll
+	darwin: { bytes: [0xcf, 0xfa, 0xed, 0xfe], name: "Mach-O 64" }, // .dylib
+	linux: { bytes: [0x7f, 0x45, 0x4c, 0x46], name: "ELF" }, //        .so
+};
+
+function run(cmd, args, opts = {}) {
+	return spawnSync(cmd, args, { stdio: "inherit", ...opts });
+}
+
+function tarBin() {
+	if (process.platform !== "win32") return "tar";
+	const sys32 = path.join(process.env.SystemRoot ?? "C:\\Windows", "System32", "tar.exe");
+	return fs.existsSync(sys32) ? sys32 : "tar";
+}
+
+function extract(archive, destDir) {
+	fs.mkdirSync(destDir, { recursive: true });
+	// Run from destDir with a bare filename: given an absolute Windows path, tar
+	// reads "C:\..." as host:path and tries to resolve a host called C.
+	const r = run(tarBin(), [archive.endsWith(".zip") ? "-xf" : "-xzf", path.basename(archive)], {
+		cwd: destDir,
+	});
+	if (r.status !== 0) throw new Error(`tar failed to extract ${path.basename(archive)}`);
+}
+
+/** Depth-first search for a file by exact basename. */
+function find(dir, name) {
+	for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+		const p = path.join(dir, entry.name);
+		if (entry.isDirectory()) {
+			const hit = find(p, name);
+			if (hit) return hit;
+		} else if (entry.name === name) {
+			return p;
+		}
+	}
+	return null;
+}
+
+/**
+ * Refuses anything that is not the MIT ONNX Runtime we pinned.
+ *
+ * Three independent checks, because the digest alone only proves we got the archive
+ * we asked for — it says nothing about having lifted the RIGHT FILE out of it, which
+ * is where a re-pin actually goes wrong (a renamed member silently vendors a 20 KB
+ * provider stub, and the failure surfaces as "the effect does nothing" months later).
+ *
+ *   1. the archive's LICENSE really is MIT — asset names are not evidence;
+ *   2. the library is a binary of the expected format for the target platform;
+ *   3. it carries the pinned version string, which is what `GetVersionString()`
+ *      returns and what `ort` compares against its `api-NN` floor.
+ */
+function verify(libPath, licensePath, targetPlatform) {
+	const license = fs.readFileSync(licensePath, "utf8");
+	if (!/^MIT License/m.test(license)) {
+		throw new Error(
+			`${path.basename(licensePath)} does not begin with "MIT License".\n` +
+				"Refusing to vendor: ONNX Runtime is MIT and this app is MIT — a relicensed\n" +
+				"upstream is a decision for a human, not a build script.",
+		);
+	}
+
+	const buf = fs.readFileSync(libPath);
+	const magic = MAGIC[targetPlatform];
+	if (!magic.bytes.every((b, i) => buf[i] === b)) {
+		const got = [...buf.subarray(0, 4)].map((b) => b.toString(16).padStart(2, "0")).join(" ");
+		throw new Error(
+			`${path.basename(libPath)} is not a ${magic.name} binary (starts with ${got}).\n` +
+				"The archive layout probably changed under the pin — check `member`.",
+		);
+	}
+
+	// The version lives in the binary as a plain NUL-terminated string.
+	if (!buf.includes(Buffer.from(`\0${VERSION}\0`, "latin1"))) {
+		throw new Error(
+			`${path.basename(libPath)} does not carry the version string ${VERSION}.\n` +
+				"Either the pin and the digest disagree, or the wrong member was extracted.",
+		);
+	}
+
+	return `MIT ONNX Runtime ${VERSION}, ${magic.name}, ${(buf.length / 1048576).toFixed(1)} MB`;
+}
+
+async function download(spec) {
+	const asset = assetName(spec);
+	const tmp = fs.mkdtempSync(path.join(os.tmpdir(), "openscreen-ort-"));
+	console.log(`Downloading ${asset}\n  from v${VERSION}`);
+	const res = await fetch(`${BASE}/${asset}`);
+	if (!res.ok) throw new Error(`Download failed: ${res.status} ${res.statusText}`);
+	const bytes = Buffer.from(await res.arrayBuffer());
+
+	// Before opening it: is this the exact artifact we pinned?
+	const got = crypto.createHash("sha256").update(bytes).digest("hex");
+	if (got !== spec.sha256) {
+		fs.rmSync(tmp, { recursive: true, force: true });
+		throw new Error(
+			`SHA-256 mismatch for ${asset}\n  expected ${spec.sha256}\n  got      ${got}\n` +
+				"Refusing to extract. Either the pin is stale or the artifact changed under it.",
+		);
+	}
+	console.log(`  sha256 ok (${(bytes.length / 1048576).toFixed(0)} MB)`);
+
+	const archive = path.join(tmp, asset);
+	fs.writeFileSync(archive, bytes);
+	extract(archive, tmp);
+	return tmp;
+}
+
+async function main() {
+	// `--target` exists for CI, which provisions for the runner it is on; without it
+	// the host is the target, which is what every local build wants.
+	const targetArg = process.argv.find((a) => a.startsWith("--target="));
+	const tag = targetArg
+		? targetArg.slice("--target=".length)
+		: `${process.platform}-${process.arch}`;
+	const [targetPlatform] = tag.split("-");
+
+	// Not an error, and deliberately exit 0: `build:mac` runs on an Intel runner for
+	// the x64 DMG, and there is no upstream library to give it. Failing here would
+	// break a release build over a feature that is designed to be absent gracefully.
+	if (tag === "darwin-x64") {
+		console.log(
+			"ONNX Runtime is not provisioned for darwin-x64: Microsoft publishes no\n" +
+				"osx-x86_64 (or universal) asset for 1.27 or later — arm64 is the only macOS\n" +
+				"target. The webcam background effects are therefore OFF on Intel Macs; the\n" +
+				"compositor logs one line and draws the camera unsegmented. Nothing else changes.",
+		);
+		return;
+	}
+
+	const spec = PINNED[tag];
+	if (!spec) {
+		console.log(
+			`No pinned ONNX Runtime for ${tag} — skipping. Have: ${Object.keys(PINNED).join(", ")}`,
+		);
+		return;
+	}
+	if (!MAGIC[targetPlatform]) {
+		throw new Error(`Unknown target platform in --target=${tag}`);
+	}
+
+	const binDir = path.join(ROOT, "electron", "native", "bin", tag);
+	const dest = path.join(binDir, spec.out);
+
+	if (fs.existsSync(dest) && !process.argv.includes("--force")) {
+		// Re-verify rather than trusting the filename: this directory is gitignored
+		// scratch space that a half-finished run or a hand copy can leave anything in.
+		const buf = fs.readFileSync(dest);
+		const magic = MAGIC[targetPlatform];
+		const looksRight =
+			magic.bytes.every((b, i) => buf[i] === b) &&
+			buf.includes(Buffer.from(`\0${VERSION}\0`, "latin1"));
+		if (looksRight) {
+			console.log(`Already present: ${dest}`);
+			console.log(`  ONNX Runtime ${VERSION}, ${(buf.length / 1048576).toFixed(1)} MB`);
+			console.log("Use --force to re-download.");
+			return;
+		}
+		console.log(`Present but not ONNX Runtime ${VERSION} — re-fetching: ${dest}`);
+	}
+
+	const tmp = await download(spec);
+	try {
+		const lib = find(tmp, spec.member);
+		if (!lib) throw new Error(`${spec.member} not found inside ${assetName(spec)}`);
+		const license = find(tmp, "LICENSE");
+		if (!license) throw new Error(`LICENSE not found inside ${assetName(spec)}`);
+
+		// Verify BEFORE vendoring: nothing unchecked reaches electron/native/bin,
+		// where the packager would happily ship it.
+		console.log("Verifying...");
+		const banner = verify(lib, license, targetPlatform);
+
+		fs.mkdirSync(binDir, { recursive: true });
+		fs.copyFileSync(lib, dest);
+		if (targetPlatform !== "win32") fs.chmodSync(dest, 0o755);
+
+		console.log(`  ${banner}`);
+		console.log(`\nVendored -> ${dest}`);
+	} finally {
+		fs.rmSync(tmp, { recursive: true, force: true });
+	}
+}
+
+main().catch((err) => {
+	console.error(`\n${err.message}`);
+	process.exit(1);
+});