diff --git a/.github/actions/install-mesa/action.yml b/.github/actions/install-mesa/action.yml new file mode 100644 index 00000000..75848168 --- /dev/null +++ b/.github/actions/install-mesa/action.yml @@ -0,0 +1,21 @@ +name: 'Install Mesa' +description: "Install the runner release's lavapipe and select it as the only Vulkan driver" +runs: + using: 'composite' + steps: + # Distro Mesa rather than gfx-rs/ci-build: those builds have no window-system support + # (-Dplatforms=), so they can't create the X11 surfaces sketches render to. + - name: Install Mesa + shell: bash + run: | + set -e + sudo apt-get install -y --no-install-recommends mesa-vulkan-drivers libvulkan1 + icd=$(dpkg -L mesa-vulkan-drivers | grep -E '/lvp_icd[^/]*\.json$' | head -1) + if [ -z "$icd" ]; then + echo "::error::no lavapipe ICD in mesa-vulkan-drivers" + dpkg -L mesa-vulkan-drivers | grep -E '\.json$' + exit 1 + fi + echo "Using $icd" + echo "VK_DRIVER_FILES=$icd" >> "$GITHUB_ENV" + dpkg-query -W mesa-vulkan-drivers diff --git a/.github/workflows/visual-comment.yml b/.github/workflows/visual-comment.yml new file mode 100644 index 00000000..b8503469 --- /dev/null +++ b/.github/workflows/visual-comment.yml @@ -0,0 +1,65 @@ +name: Visual comment + +# Posts visual.yml's PR summary as a sticky comment. Runs from the base repository so it can +# write to PRs from forks; it never checks out PR code and treats the artifact as untrusted. + +on: + workflow_run: + workflows: [ Visual ] + types: [ completed ] + +permissions: {} + +jobs: + comment: + if: >- + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion != 'cancelled' && + github.event.workflow_run.conclusion != 'skipped' + runs-on: ubuntu-latest + permissions: + actions: read + pull-requests: write + steps: + - uses: actions/download-artifact@v8 + id: download + continue-on-error: true + with: + name: visual-comment + path: comment + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Validate + id: pr + if: steps.download.outcome == 'success' + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + pr=$(cat comment/pr) + notable=$(cat comment/notable) + [[ "$pr" =~ ^[0-9]+$ ]] || { echo "invalid PR number"; exit 1; } + [[ "$notable" == true || "$notable" == false ]] || { echo "invalid notable flag"; exit 1; } + pr_head=$(gh api "repos/$GITHUB_REPOSITORY/pulls/$pr" --jq .head.sha) + [[ "$pr_head" == "$HEAD_SHA" ]] || { echo "PR #$pr is not at $HEAD_SHA; skipping"; exit 0; } + echo "number=$pr" >> "$GITHUB_OUTPUT" + echo "notable=$notable" >> "$GITHUB_OUTPUT" + + - name: Post comment + if: steps.pr.outputs.notable == 'true' + uses: marocchino/sticky-pull-request-comment@v3.0.5 + with: + header: visual-regression + number_force: ${{ steps.pr.outputs.number }} + path: comment/comment.md + + # A clean run only refreshes an earlier comment, so passing PRs stay quiet. + - name: Update comment + if: steps.pr.outputs.notable == 'false' + uses: marocchino/sticky-pull-request-comment@v3.0.5 + with: + header: visual-regression + number_force: ${{ steps.pr.outputs.number }} + path: comment/comment.md + only_update: true diff --git a/.github/workflows/visual.yml b/.github/workflows/visual.yml new file mode 100644 index 00000000..2a2f8174 --- /dev/null +++ b/.github/workflows/visual.yml @@ -0,0 +1,145 @@ +name: Visual + +# Renders tests/visual/cases.toml under lavapipe + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + types: [ opened, synchronize, reopened, labeled, unlabeled ] + +permissions: + contents: read + actions: read + +concurrency: + group: visual-${{ github.event.pull_request.number || github.sha }} + cancel-in-progress: true + +env: + CARGO_TERM_COLOR: always + CARGO_INCREMENTAL: 0 + CARGO_PROFILE_RELEASE_DEBUG: 0 + WGPU_BACKEND: vulkan + WGPU_ADAPTER_NAME: llvmpipe + # Catch reads of uninitialized GPU memory instead of rendering whatever was there + LVP_POISON_MEMORY: "true" + # llvmpipe JITs for the host CPU + LP_NATIVE_VECTOR_WIDTH: "256" + ODIFF_VERSION: 4.5.0 + +jobs: + visual: + name: Visual regression + # Only re-run on label changes that involve the override label. + if: >- + github.event_name == 'push' || + (github.event.action != 'labeled' && github.event.action != 'unlabeled') || + github.event.label.name == 'deliberate-rendering-change' + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + # PR checkouts are merge commits; HEAD^1 is the main commit they're compared against. + fetch-depth: 2 + persist-credentials: false + + - uses: ./.github/actions/setup + + - uses: ./.github/actions/install-mesa + + - uses: astral-sh/setup-uv@v6 + + - name: Install xvfb and odiff + run: | + sudo apt-get install -y --no-install-recommends xvfb + npm install -g "odiff-bin@$ODIFF_VERSION" + + - uses: actions/cache@v4 + with: + path: | + ~/.cargo/bin/ + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + target/ + key: ${{ runner.os }}-cargo-visual-${{ hashFiles('**/Cargo.lock') }} + restore-keys: | + ${{ runner.os }}-cargo-visual- + ${{ runner.os }}-cargo- + + - name: Render + run: xvfb-run -a -s "-screen 0 1920x1080x24" python3 tests/visual/visual.py render --out out/actual + + - uses: actions/upload-artifact@v7 + with: + name: visual-screenshots + path: out/actual + retention-days: ${{ github.event_name == 'push' && 90 || 14 }} + + - name: Fetch baseline from main + if: github.event_name == 'pull_request' + env: + GH_TOKEN: ${{ github.token }} + run: | + sha=$(git rev-parse HEAD^1) + latest() { + gh run list -R "$GITHUB_REPOSITORY" --workflow visual.yml --branch main --event push \ + --commit "$sha" --limit 1 --json databaseId,status,conclusion --jq '.[0] // empty' + } + run=$(latest) + if [ -n "$run" ] && [ "$(jq -r .status <<< "$run")" != completed ]; then + id=$(jq -r .databaseId <<< "$run") + echo "Waiting for main's run $id to publish screenshots" + gh run watch "$id" -R "$GITHUB_REPOSITORY" --interval 30 > /dev/null || true + run=$(latest) + fi + if [ "$(jq -r '.conclusion // empty' <<< "$run")" = success ] && + gh run download "$(jq -r .databaseId <<< "$run")" -R "$GITHUB_REPOSITORY" -n visual-screenshots -D out/baseline; then + echo "Using screenshots from run $(jq -r .databaseId <<< "$run")" + else + echo "::notice::No screenshots published for main at $sha; cases are reported as new" + fi + + - name: Compare + id: compare + if: github.event_name == 'pull_request' + env: + ALLOW_CHANGES: ${{ contains(github.event.pull_request.labels.*.name, 'deliberate-rendering-change') && '--allow-changes' || '' }} + run: | + mkdir -p out/baseline + python3 tests/visual/visual.py compare --baseline out/baseline --actual out/actual --out out/report $ALLOW_CHANGES + + - uses: actions/upload-artifact@v7 + id: report + if: always() && hashFiles('out/report/report.html') != '' + with: + path: out/report/report.html + archive: false + retention-days: 14 + + - name: Summarize + if: always() && hashFiles('out/report/summary.md') != '' + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + REPORT_URL: ${{ steps.report.outputs.artifact-url }} + run: | + mkdir -p out/comment + { + cat out/report/summary.md + echo + echo "[Open the visual report]($REPORT_URL) · [workflow run]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID)" + } > out/comment/comment.md + cat out/comment/comment.md >> "$GITHUB_STEP_SUMMARY" + echo "$PR_NUMBER" > out/comment/pr + jq '[.results[] | select(.status != "pass")] | length > 0' out/report/results.json > out/comment/notable + + - uses: actions/upload-artifact@v7 + if: always() && hashFiles('out/comment/comment.md') != '' + with: + name: visual-comment + path: out/comment + retention-days: 1 diff --git a/Cargo.lock b/Cargo.lock index c6b5e7ff..350f0e5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6231,6 +6231,7 @@ dependencies = [ "objc2 0.6.4", "objc2-app-kit 0.3.2", "parley 0.7.0", + "png", "processing_core", "raw-window-handle", "skrifa 0.37.0", diff --git a/Cargo.toml b/Cargo.toml index 2bae50b6..be9f10df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -94,7 +94,7 @@ processing_glfw = { workspace = true } rand = { workspace = true } [target.'cfg(target_os = "linux")'.dev-dependencies] -processing_glfw = { workspace = true, features = ["wayland"] } +processing_glfw = { workspace = true, features = ["wayland", "x11"] } ## TODO: Remove these patches once we've moved back to depending on upstream bevy [patch."https://github.com/bevyengine/bevy"] diff --git a/crates/processing_glfw/src/lib.rs b/crates/processing_glfw/src/lib.rs index 9bf6a2f2..a6a30891 100644 --- a/crates/processing_glfw/src/lib.rs +++ b/crates/processing_glfw/src/lib.rs @@ -17,6 +17,10 @@ use processing_input::{ }; use processing_render::surface::{MonitorWorkarea, WindowControls}; +fn is_wayland(glfw: &Glfw) -> bool { + cfg!(target_os = "linux") && glfw.get_platform() == glfw::Platform::Wayland +} + /// A single GLFW instance drives every window (GLFW's event pump is global). The /// main window is `windows[0]`; `create_window` appends more. pub struct GlfwContext { @@ -96,30 +100,33 @@ impl GlfwContext { // Set _NET_WM_WINDOW_TYPE_DIALOG so tiling WMs (i3, sway) float the window #[cfg(all(target_os = "linux", feature = "x11"))] - unsafe { + if glfw.get_platform() == glfw::Platform::X11 { use std::ffi::CString; - let display = window.glfw.get_x11_display() as *mut x11::xlib::Display; - let xwindow = window.get_x11_window() as x11::xlib::Window; - let net_wm_window_type = x11::xlib::XInternAtom( - display, - CString::new("_NET_WM_WINDOW_TYPE").unwrap().as_ptr(), - 0, - ); - let net_wm_window_type_dialog = x11::xlib::XInternAtom( - display, - CString::new("_NET_WM_WINDOW_TYPE_DIALOG").unwrap().as_ptr(), - 0, - ); - x11::xlib::XChangeProperty( - display, - xwindow, - net_wm_window_type, - x11::xlib::XA_ATOM, - 32, - x11::xlib::PropModeReplace, - &net_wm_window_type_dialog as *const _ as *const u8, - 1, - ); + // SAFETY: GLFW is on X11, so the display and window handles are live Xlib objects. + unsafe { + let display = window.glfw.get_x11_display() as *mut x11::xlib::Display; + let xwindow = window.get_x11_window() as x11::xlib::Window; + let net_wm_window_type = x11::xlib::XInternAtom( + display, + CString::new("_NET_WM_WINDOW_TYPE").unwrap().as_ptr(), + 0, + ); + let net_wm_window_type_dialog = x11::xlib::XInternAtom( + display, + CString::new("_NET_WM_WINDOW_TYPE_DIALOG").unwrap().as_ptr(), + 0, + ); + x11::xlib::XChangeProperty( + display, + xwindow, + net_wm_window_type, + x11::xlib::XA_ATOM, + 32, + x11::xlib::PropModeReplace, + &net_wm_window_type_dialog as *const _ as *const u8, + 1, + ); + } } window.show(); @@ -275,19 +282,41 @@ impl GlfwContext { let handle = self.windows[idx].window.get_win32_window() as u64; surface_create_windows(handle, width, height, scale_factor, transparent)? }; - #[cfg(all(target_os = "linux", feature = "wayland"))] - let entity = { - use processing_render::surface_create_wayland; - let wh = self.windows[idx].window.get_wayland_window() as u64; - let dh = self.glfw.get_wayland_display() as u64; - surface_create_wayland(wh, dh, width, height, scale_factor, transparent)? - }; - #[cfg(all(target_os = "linux", feature = "x11", not(feature = "wayland")))] - let entity = { - use processing_render::surface_create_x11; - let wh = self.windows[idx].window.get_x11_window() as u64; - let dh = self.glfw.get_x11_display() as u64; - surface_create_x11(wh, dh, width, height, scale_factor, transparent)? + #[cfg(target_os = "linux")] + let entity = match self.glfw.get_platform() { + #[cfg(feature = "wayland")] + glfw::Platform::Wayland => { + let wh = self.windows[idx].window.get_wayland_window() as u64; + let dh = self.glfw.get_wayland_display() as u64; + processing_render::surface_create_wayland( + wh, + dh, + width, + height, + scale_factor, + transparent, + )? + } + #[cfg(feature = "x11")] + glfw::Platform::X11 => { + let wh = self.windows[idx].window.get_x11_window() as u64; + let dh = self.glfw.get_x11_display() as u64; + processing_render::surface_create_x11( + wh, + dh, + width, + height, + scale_factor, + transparent, + )? + } + platform => { + return Err(processing_core::error::ProcessingError::InvalidArgument( + format!( + "GLFW is running on {platform:?}, which this build doesn't support; enable the matching `x11`/`wayland` feature" + ), + )); + } }; self.windows[idx].surface = Some(entity); @@ -319,7 +348,7 @@ impl GlfwContext { if input_flush().is_err() { return false; } - main_open + main_open && !processing_render::ci::done() } /// Content scale (DPI) of the main window. @@ -438,8 +467,7 @@ impl ManagedWindow { if desired.maximize { self.window.maximize(); } - #[cfg(not(all(target_os = "linux", feature = "wayland")))] - if desired.focus { + if desired.focus && !is_wayland(glfw) { self.window.focus(); } @@ -463,18 +491,15 @@ impl ManagedWindow { self.last_applied.size = bevy::math::UVec2::new(w.max(0) as u32, h.max(0) as u32); } - #[cfg(not(feature = "wayland"))] fn frame_pos(&self) -> IVec2 { + if is_wayland(&self.window.glfw) { + return self.last_applied.position; + } let (cx, cy) = self.window.get_pos(); let (inset_l, inset_t, _, _) = self.window.get_frame_size(); IVec2::new(cx - inset_l, cy - inset_t) } - #[cfg(feature = "wayland")] - fn frame_pos(&self) -> IVec2 { - self.last_applied.position - } - fn apply_window(&mut self, glfw: &mut Glfw, desired: &DesiredWindow) { let last = &mut self.last_applied; @@ -482,9 +507,9 @@ impl ManagedWindow { self.window.set_title(&desired.title); last.title.clone_from(&desired.title); } - #[cfg(not(feature = "wayland"))] if let Some(pos) = desired.position && pos != last.position + && !is_wayland(glfw) { let (inset_l, inset_t, _, _) = self.window.get_frame_size(); self.window.set_pos(pos.x + inset_l, pos.y + inset_t); @@ -512,16 +537,18 @@ impl ManagedWindow { last.decorations = desired.decorations; } if desired.window_level != last.window_level { - #[cfg(not(all(target_os = "linux", feature = "wayland")))] - self.window - .set_floating(matches!(desired.window_level, BevyWindowLevel::AlwaysOnTop)); + if !is_wayland(glfw) { + self.window + .set_floating(matches!(desired.window_level, BevyWindowLevel::AlwaysOnTop)); + } last.window_level = desired.window_level; } if let Some(opacity) = desired.opacity && (opacity - last.opacity).abs() > f32::EPSILON { - #[cfg(not(all(target_os = "linux", feature = "wayland")))] - self.window.set_opacity(opacity); + if !is_wayland(glfw) { + self.window.set_opacity(opacity); + } last.opacity = opacity; } if desired.fullscreen_on != last.fullscreen_on { @@ -590,7 +617,6 @@ impl ManagedWindow { #[derive(Clone, Debug)] struct DesiredWindow { title: String, - #[cfg(not(feature = "wayland"))] position: Option, size: bevy::math::UVec2, visible: bool, @@ -602,7 +628,6 @@ struct DesiredWindow { iconify: bool, restore: bool, maximize: bool, - #[cfg(not(all(target_os = "linux", feature = "wayland")))] focus: bool, } @@ -624,7 +649,6 @@ fn read_desired_window(surface: Entity) -> Option { }; Ok(Some(DesiredWindow { title: window.title.clone(), - #[cfg(not(feature = "wayland"))] position: match window.position { WindowPosition::At(p) => Some(p), _ => None, @@ -642,7 +666,6 @@ fn read_desired_window(surface: Entity) -> Option { iconify: controls.pending_iconify, restore: controls.pending_restore, maximize: controls.pending_maximize, - #[cfg(not(all(target_os = "linux", feature = "wayland")))] focus: controls.pending_focus, })) }) diff --git a/crates/processing_input/src/lib.rs b/crates/processing_input/src/lib.rs index 4ee0bfd4..118a295f 100644 --- a/crates/processing_input/src/lib.rs +++ b/crates/processing_input/src/lib.rs @@ -8,6 +8,7 @@ use bevy::input::mouse::{ }; use bevy::input::touch::TouchPhase; use bevy::prelude::*; +use bevy::time::TimeSystems; use bevy::window::{CursorMoved, WindowResized}; use processing_core::app_mut; @@ -17,10 +18,18 @@ pub use state::{CursorPosition, LastKey, LastMouseButton}; pub struct InputPlugin; +#[derive(Resource, Default)] +struct InputFlushing(bool); + impl Plugin for InputPlugin { fn build(&self, app: &mut App) { app.init_resource::() .init_resource::() + .init_resource::() + .configure_sets( + First, + TimeSystems.run_if(|flushing: Res| !flushing.0), + ) .add_systems( PreUpdate, ( @@ -208,9 +217,11 @@ pub fn input_set_cursor_icon( pub fn input_flush() -> error::Result<()> { app_mut(|app| { let world = app.world_mut(); + world.resource_mut::().0 = true; world.run_schedule(First); world.run_schedule(PreUpdate); world.run_schedule(RunFixedMainLoop); + world.resource_mut::().0 = false; Ok(()) }) } diff --git a/crates/processing_pyo3/Cargo.toml b/crates/processing_pyo3/Cargo.toml index 0b8ab77d..ce5c3449 100644 --- a/crates/processing_pyo3/Cargo.toml +++ b/crates/processing_pyo3/Cargo.toml @@ -11,7 +11,7 @@ name = "mewnala" crate-type = ["cdylib", "rlib"] [features] -default = ["wayland", "static-link"] +default = ["wayland", "x11", "static-link"] wayland = ["processing/wayland", "processing_glfw/wayland"] static-link = ["processing_glfw/static-link"] x11 = ["processing/x11", "processing_glfw/x11"] diff --git a/crates/processing_pyo3/src/lib.rs b/crates/processing_pyo3/src/lib.rs index a21a2f59..32eb77c0 100644 --- a/crates/processing_pyo3/src/lib.rs +++ b/crates/processing_pyo3/src/lib.rs @@ -493,6 +493,9 @@ pub mod mewnala { #[pymodule_init] fn init(module: &Bound<'_, PyModule>) -> PyResult<()> { + if processing_render::ci::enabled() { + PyModule::import(module.py(), "random")?.call_method1("seed", (0,))?; + } super::constants::register(module) } diff --git a/crates/processing_pyo3/src/math.rs b/crates/processing_pyo3/src/math.rs index 81bc61cf..8f86c519 100644 --- a/crates/processing_pyo3/src/math.rs +++ b/crates/processing_pyo3/src/math.rs @@ -1,3 +1,4 @@ +use std::cell::RefCell; use std::hash::{Hash, Hasher}; use bevy::math::{Affine2, EulerRot, Mat2, Quat, Vec2, Vec3, Vec4}; @@ -6,6 +7,20 @@ use pyo3::{ prelude::*, types::PyTuple, }; +use rand::{SeedableRng, rngs::StdRng}; +use rand_distr::Distribution; + +thread_local! { + static CI_RNG: RefCell> = + RefCell::new(processing_render::ci::enabled().then(|| StdRng::seed_from_u64(0))); +} + +fn sample(dist: impl Distribution) -> T { + CI_RNG.with_borrow_mut(|rng| match rng { + Some(rng) => dist.sample(rng), + None => dist.sample(&mut rand::rng()), + }) +} pub fn hash_f32(val: f32, state: &mut impl Hasher) { if val == 0.0 { @@ -537,8 +552,7 @@ impl_py_vec!(PyVec2, "Vec2", 2, [(x, set_x, 0), (y, set_y, 1)], Vec2, extra { #[staticmethod] fn random() -> Self { - use rand_distr::{Distribution, UnitCircle}; - let [x, y]: [f32; 2] = UnitCircle.sample(&mut rand::rng()); + let [x, y]: [f32; 2] = sample(rand_distr::UnitCircle); Self(Vec2::new(x, y)) } @@ -560,8 +574,7 @@ impl_py_vec!(PyVec3, "Vec3", 3, [(x, set_x, 0), (y, set_y, 1), (z, set_z, 2)], V #[staticmethod] fn random() -> Self { - use rand_distr::{Distribution, UnitSphere}; - let [x, y, z]: [f32; 3] = UnitSphere.sample(&mut rand::rng()); + let [x, y, z]: [f32; 3] = sample(rand_distr::UnitSphere); Self(Vec3::new(x, y, z)) } diff --git a/crates/processing_render/Cargo.toml b/crates/processing_render/Cargo.toml index 146a9bbe..6dddeabe 100644 --- a/crates/processing_render/Cargo.toml +++ b/crates/processing_render/Cargo.toml @@ -27,6 +27,7 @@ notosans = "0.1" raw-window-handle = "0.6" half = "2.7" crossbeam-channel = "0.5" +png = "0.18" processing_core = { workspace = true } [build-dependencies] diff --git a/crates/processing_render/src/ci.rs b/crates/processing_render/src/ci.rs new file mode 100644 index 00000000..c929c39e --- /dev/null +++ b/crates/processing_render/src/ci.rs @@ -0,0 +1,178 @@ +//! Deterministic screenshot capture for visual regression CI. +use std::path::PathBuf; +use std::time::Duration; + +use bevy::{ + camera::RenderTarget, platform::time::Instant, prelude::*, + render::render_resource::TextureFormat, time::TimeUpdateStrategy, +}; +use processing_core::app_mut; +use processing_core::error::{ProcessingError, Result}; + +pub const SCREENSHOT_ENV: &str = "PROCESSING_CI_SCREENSHOT"; +pub const FRAME_ENV: &str = "PROCESSING_CI_FRAME"; +pub const DEFAULT_FRAME: u32 = 10; +pub const FIXED_TIMESTEP: Duration = Duration::from_micros(16_667); + +#[derive(Resource, Debug)] +pub struct CiCapture { + path: PathBuf, + frame: u32, + epoch: Instant, + target: Option, + frames_ended: u32, + done: bool, +} + +pub fn enabled() -> bool { + std::env::var_os(SCREENSHOT_ENV).is_some_and(|p| !p.is_empty()) +} + +impl CiCapture { + fn from_env() -> Result> { + let Some(path) = std::env::var_os(SCREENSHOT_ENV).filter(|p| !p.is_empty()) else { + return Ok(None); + }; + let frame = match std::env::var(FRAME_ENV) { + Ok(s) => s.parse::().ok().filter(|n| *n > 0).ok_or_else(|| { + ProcessingError::InvalidArgument(format!( + "{FRAME_ENV} must be a positive integer, got {s:?}" + )) + })?, + Err(_) => DEFAULT_FRAME, + }; + Ok(Some(Self { + path: path.into(), + frame, + epoch: Instant::now(), + target: None, + frames_ended: 0, + done: false, + })) + } +} + +pub struct CiPlugin; + +impl Plugin for CiPlugin { + fn build(&self, app: &mut App) { + match CiCapture::from_env() { + Ok(Some(capture)) => { + info!( + "CI capture enabled: frame {} -> {}", + capture.frame, + capture.path.display() + ); + let epoch = capture.epoch; + app.insert_resource(capture) + .insert_resource(TimeUpdateStrategy::ManualInstant(epoch)); + } + Ok(None) => {} + Err(e) => panic!("{e}"), + } + } +} + +pub(crate) fn after_end_draw(app: &mut App, entity: Entity) -> Result<()> { + let world = app.world_mut(); + let Some(capture) = world.get_resource::() else { + return Ok(()); + }; + if capture.done { + return Ok(()); + } + match capture.target { + Some(target) if target != entity => return Ok(()), + Some(_) => {} + None => { + let is_window = matches!( + world.get::(entity), + Some(RenderTarget::Window(_)) + ); + if !is_window { + return Ok(()); + } + world.resource_mut::().target = Some(entity); + } + } + + let mut capture = world.resource_mut::(); + capture.frames_ended += 1; + let now = capture.epoch + FIXED_TIMESTEP * capture.frames_ended; + let reached = capture.frames_ended >= capture.frame; + world.insert_resource(TimeUpdateStrategy::ManualInstant(now)); + if !reached { + return Ok(()); + } + let mut capture = world.resource_mut::(); + let path = capture.path.clone(); + capture.done = true; + + let (width, height, rgba) = readback_srgba8(app, entity)?; + write_png(&path, width, height, &rgba)?; + info!("CI capture written to {}", path.display()); + Ok(()) +} + +pub fn done() -> bool { + app_mut(|app| { + Ok(app + .world() + .get_resource::() + .is_some_and(|c| c.done)) + }) + .unwrap_or(false) +} + +pub(crate) fn readback_srgba8(app: &mut App, entity: Entity) -> Result<(u32, u32, Vec)> { + crate::graphics::flush(app, entity)?; + let vt = crate::graphics::view_target(app, entity)?; + let texture = vt.main_texture().clone(); + let raw = app + .world_mut() + .run_system_cached_with(crate::graphics::readback_raw, (entity, texture)) + .unwrap()?; + let rgba = match raw.format { + TextureFormat::Rgba8UnormSrgb => raw.bytes, + format => { + let px_size = crate::image::pixel_size(format)?; + crate::image::bytes_to_pixels( + &raw.bytes, + format, + raw.width, + raw.height, + raw.width as usize * px_size, + )? + .iter() + .flat_map(|pixel| Srgba::from(*pixel).to_u8_array()) + .collect() + } + }; + Ok((raw.width, raw.height, rgba)) +} + +pub(crate) fn write_png( + path: &std::path::Path, + width: u32, + height: u32, + rgba: &[u8], +) -> Result<()> { + let io_err = |e: std::io::Error| { + ProcessingError::InvalidArgument(format!("write {}: {e}", path.display())) + }; + if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) { + std::fs::create_dir_all(parent).map_err(io_err)?; + } + let file = std::fs::File::create(path).map_err(io_err)?; + let mut encoder = png::Encoder::new(std::io::BufWriter::new(file), width, height); + encoder.set_color(png::ColorType::Rgba); + encoder.set_depth(png::BitDepth::Eight); + encoder.set_source_srgb(png::SrgbRenderingIntent::Perceptual); + let png_err = + |e: png::EncodingError| ProcessingError::InvalidArgument(format!("PNG encode: {e}")); + encoder + .write_header() + .map_err(png_err)? + .write_image_data(rgba) + .map_err(png_err) +} diff --git a/crates/processing_render/src/graphics.rs b/crates/processing_render/src/graphics.rs index ae09bd22..0f0dea8a 100644 --- a/crates/processing_render/src/graphics.rs +++ b/crates/processing_render/src/graphics.rs @@ -581,7 +581,8 @@ pub fn present(app: &mut App, entity: Entity) -> Result<()> { /// End the current draw pub fn end_draw(app: &mut App, entity: Entity) -> Result<()> { - present(app, entity) + present(app, entity)?; + crate::ci::after_end_draw(app, entity) } /// Do some work on the GPU to ensure that the render target texture is initialized and can be read diff --git a/crates/processing_render/src/lib.rs b/crates/processing_render/src/lib.rs index 1113411e..8634690c 100644 --- a/crates/processing_render/src/lib.rs +++ b/crates/processing_render/src/lib.rs @@ -1,6 +1,7 @@ #![allow(clippy::module_inception)] pub mod camera; +pub mod ci; pub mod color; pub mod compute; pub mod geometry; @@ -101,7 +102,8 @@ impl Plugin for ProcessingRenderPlugin { bevy::camera_controller::free_camera::FreeCameraPlugin, bevy::camera_controller::pan_camera::PanCameraPlugin, text::font::TextPlugin, - )); + )) + .add_plugins(ci::CiPlugin); app.add_systems(First, (clear_transient_meshes, activate_cameras)) .add_systems( diff --git a/examples/particles_noise.rs b/examples/particles_noise.rs index 8c193bac..71baabc3 100644 --- a/examples/particles_noise.rs +++ b/examples/particles_noise.rs @@ -1,5 +1,4 @@ use processing_glfw::GlfwContext; -use std::time::Instant; use bevy::math::Vec3; use processing::prelude::*; @@ -53,7 +52,6 @@ fn sketch() -> error::Result<()> { }; let noise = particles_kernel_noise()?; - let start = Instant::now(); while glfw_ctx.poll_events() { graphics_begin_draw(graphics)?; graphics_record_command( @@ -71,7 +69,7 @@ fn sketch() -> error::Result<()> { )?; graphics_end_draw(graphics)?; - let t = start.elapsed().as_secs_f32(); + let t = elapsed_time()?; compute_set(noise, "scale", shader_value::ShaderValue::Float(0.25))?; compute_set(noise, "strength", shader_value::ShaderValue::Float(0.02))?; compute_set(noise, "time", shader_value::ShaderValue::Float(t * 0.5))?; diff --git a/justfile b/justfile index 4af5b1d1..6cd2a1a9 100644 --- a/justfile +++ b/justfile @@ -35,3 +35,9 @@ wasm-release: wasm-serve: wasm-build python3 -m http.server 8000 + +visual-render out="target/visual/actual" *args: + python3 tests/visual/visual.py render --out {{out}} {{args}} + +visual-compare baseline actual="target/visual/actual" out="target/visual/report": + python3 tests/visual/visual.py compare --baseline {{baseline}} --actual {{actual}} --out {{out}} diff --git a/tests/visual/cases.toml b/tests/visual/cases.toml new file mode 100644 index 00000000..9353fdaf --- /dev/null +++ b/tests/visual/cases.toml @@ -0,0 +1,178 @@ +# Visual regression cases. + +[defaults] +frame = 30 +threshold = 0.1 +max_diff_percent = 0.05 + +[[case]] +name = "rectangle" +rust = "rectangle" + +[[case]] +name = "primitives_3d" +rust = "primitives_3d" + +[[case]] +name = "particles_noise" +rust = "particles_noise" + +[[case]] +name = "py_primitives_3d" +python = "primitives_3d.py" + +[[case]] +name = "py_particles_noise" +python = "particles_noise.py" + +[[case]] +name = "py_animated_mesh" +python = "animated_mesh.py" + +[[case]] +name = "py_background_image" +python = "background_image.py" + +[[case]] +name = "py_blend_modes" +python = "blend_modes.py" + +[[case]] +name = "py_box" +python = "box.py" + +[[case]] +name = "py_camera_controllers" +python = "camera_controllers.py" + +[[case]] +name = "py_curves" +python = "curves.py" + +[[case]] +name = "py_custom_material" +python = "custom_material.py" + +[[case]] +name = "py_feedback" +python = "feedback.py" + +[[case]] +name = "py_flocking" +python = "flocking.py" + +[[case]] +name = "py_flocking_duck" +python = "flocking_duck.py" + +[[case]] +name = "py_flocking_gpu" +python = "flocking_gpu.py" + +[[case]] +name = "py_geometry_methods" +python = "geometry_methods.py" + +[[case]] +name = "py_gltf_load" +python = "gltf_load.py" + +[[case]] +name = "py_lights" +python = "lights.py" + +[[case]] +name = "py_materials" +python = "materials.py" + +[[case]] +name = "py_multi_window" +python = "multi_window.py" + +[[case]] +name = "py_particles_animated" +python = "particles_animated.py" + +[[case]] +name = "py_particles_basic" +python = "particles_basic.py" + +[[case]] +name = "py_particles_density" +python = "particles_density.py" + +[[case]] +name = "py_particles_emit" +python = "particles_emit.py" + +[[case]] +name = "py_particles_emit_gpu" +python = "particles_emit_gpu.py" + +[[case]] +name = "py_particles_from_mesh" +python = "particles_from_mesh.py" + +[[case]] +name = "py_particles_gpu_surface" +python = "particles_gpu_surface.py" + +[[case]] +name = "py_particles_gpu_surface_lit" +python = "particles_gpu_surface_lit.py" + +[[case]] +name = "py_particles_lifecycle" +python = "particles_lifecycle.py" + +[[case]] +name = "py_particles_lines" +python = "particles_lines.py" + +[[case]] +name = "py_particles_lissajous" +python = "particles_lissajous.py" + +[[case]] +name = "py_particles_points" +python = "particles_points.py" + +[[case]] +name = "py_particles_scatter_volume" +python = "particles_scatter_volume.py" + +[[case]] +name = "py_particles_sphere" +python = "particles_sphere.py" + +[[case]] +name = "py_particles_stress" +python = "particles_stress.py" + +[[case]] +name = "py_particles_surface" +python = "particles_surface.py" + +[[case]] +name = "py_primitives_2d" +python = "primitives_2d.py" + +[[case]] +name = "py_rectangle" +python = "rectangle.py" + +[[case]] +name = "py_shapes" +python = "shapes.py" + +[[case]] +name = "py_style_stack" +python = "style_stack.py" + +[[case]] +name = "py_text" +python = "text.py" + +[[case]] +name = "py_window_controls" +python = "window_controls.py" diff --git a/tests/visual/visual.py b/tests/visual/visual.py new file mode 100644 index 00000000..bd42a144 --- /dev/null +++ b/tests/visual/visual.py @@ -0,0 +1,384 @@ +#!/usr/bin/env python3 +"""Visual regression harness. + + visual.py render --out DIR [--only NAME ...] + visual.py compare --baseline DIR --actual DIR --out DIR [--allow-changes] +""" + +from __future__ import annotations + +import argparse +import base64 +import html +import json +import os +import re +import shutil +import subprocess +import sys +import time +import tomllib +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[1] +CASES_FILE = HERE / "cases.toml" +RENDER_TIMEOUT_SECS = 300 +LOG_TAIL_LINES = 40 + + +def load_cases(only: list[str] | None = None) -> list[dict]: + data = tomllib.loads(CASES_FILE.read_text()) + defaults = data.get("defaults", {}) + cases, seen = [], set() + for raw in data.get("case", []): + case = {**defaults, **raw} + name = case.get("name") + if not name or not re.fullmatch(r"[A-Za-z0-9_-]+", name): + sys.exit(f"cases.toml: invalid case name {name!r}") + if name in seen: + sys.exit(f"cases.toml: duplicate case {name!r}") + if ("rust" in case) == ("python" in case): + sys.exit(f"cases.toml: case {name!r} must set exactly one of `rust` or `python`") + seen.add(name) + cases.append(case) + if only: + unknown = set(only) - seen + if unknown: + sys.exit(f"unknown case(s): {', '.join(sorted(unknown))}") + cases = [c for c in cases if c["name"] in only] + return cases + + +def run(cmd: list[str], cwd: Path, env: dict | None = None) -> None: + print(f"$ {' '.join(cmd)} (in {cwd})", flush=True) + subprocess.run(cmd, cwd=cwd, env=env, check=True) + + +def git_head(root: Path) -> str | None: + try: + out = subprocess.run( + ["git", "rev-parse", "HEAD"], cwd=root, capture_output=True, text=True, check=True + ) + return out.stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return None + + +def adapter_from_log(log: str) -> str | None: + match = re.search(r"AdapterInfo \{[^}]*\}", log) + if not match: + return None + fields = dict(re.findall(r'(\w+): ("[^"]*"|[^,}]+)', match.group(0))) + name, backend, driver = (fields.get(k, "").strip().strip('"') for k in ("name", "backend", "driver_info")) + return f"{name} ({backend}{', ' + driver if driver else ''})" + + +def cmd_render(args: argparse.Namespace) -> int: + root = REPO + out = Path(args.out).resolve() + logs = out / "logs" + logs.mkdir(parents=True, exist_ok=True) + cases = load_cases(args.only) + + rust = [c for c in cases if "rust" in c] + python = [c for c in cases if "python" in c] + py_dir = root / "crates" / "processing_pyo3" + + if rust and not args.skip_build: + examples = [arg for c in rust for arg in ("--example", c["rust"])] + run(["cargo", "build", "--release", *examples], cwd=root) + if python and not args.skip_build: + run(["uv", "run", "maturin", "develop", "--release"], cwd=py_dir) + + manifest = {"commit": git_head(root), "adapter": None, "cases": {}} + for case in cases: + name = case["name"] + png = out / f"{name}.png" + png.unlink(missing_ok=True) + env = { + **os.environ, + "PROCESSING_CI_SCREENSHOT": str(png), + "PROCESSING_CI_FRAME": str(case["frame"]), + } + env.setdefault("PROCESSING_ASSET_ROOT", str(root / "assets")) + if "rust" in case: + cmd, cwd = ["cargo", "run", "--quiet", "--release", "--example", case["rust"]], root + else: + cmd, cwd = ["uv", "run", "python", f"examples/{case['python']}"], py_dir + + print(f"--- {name}", flush=True) + started = time.monotonic() + try: + proc = subprocess.run( + cmd, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + errors="replace", + timeout=RENDER_TIMEOUT_SECS, + ) + log, code = proc.stdout, proc.returncode + except subprocess.TimeoutExpired as e: + log = (e.stdout or b"").decode(errors="replace") if isinstance(e.stdout, bytes) else (e.stdout or "") + log += f"\n[visual.py] timed out after {RENDER_TIMEOUT_SECS}s\n" + code = None + elapsed = time.monotonic() - started + (logs / f"{name}.log").write_text(log) + + if code == 0 and png.exists(): + status = "ok" + elif code == 0: + status = "no-capture" + else: + status = "timeout" if code is None else f"exit {code}" + manifest["adapter"] = manifest["adapter"] or adapter_from_log(log) + manifest["cases"][name] = {"status": status, "seconds": round(elapsed, 1)} + print(f" {status} in {elapsed:.1f}s", flush=True) + if status != "ok": + print("\n".join(log.splitlines()[-LOG_TAIL_LINES:]), flush=True) + + (out / "manifest.json").write_text(json.dumps(manifest, indent=2)) + failed = [n for n, c in manifest["cases"].items() if c["status"] != "ok"] + if failed: + print(f"failed to render: {', '.join(failed)}", file=sys.stderr) + # Render failures are recorded in the manifest and surfaced by `compare`. + return 0 + + +def odiff(exe: str, base: Path, actual: Path, diff: Path, threshold: float) -> tuple[str, float]: + """Returns (outcome, differing percent) where outcome is match, pixels or layout.""" + proc = subprocess.run( + [ + exe, str(base), str(actual), str(diff), + "--antialiasing", "--fail-on-layout", "--parsable-stdout", "--diff-mask", + f"--threshold={threshold}", + ], + capture_output=True, + text=True, + ) + stdout = proc.stdout.strip() + if proc.returncode == 0: + return "match", 0.0 + if proc.returncode == 21: + return "layout", 100.0 + if proc.returncode == 22: + _count, percent = stdout.split(";") + return "pixels", float(percent) + raise RuntimeError(f"odiff exited {proc.returncode}: {stdout} {proc.stderr.strip()}") + + +def read_manifest(directory: Path) -> dict: + path = directory / "manifest.json" + return json.loads(path.read_text()) if path.exists() else {"cases": {}} + + +def cmd_compare(args: argparse.Namespace) -> int: + baseline, actual, out = Path(args.baseline), Path(args.actual), Path(args.out) + diffs = out / "diff" + diffs.mkdir(parents=True, exist_ok=True) + exe = args.odiff or shutil.which("odiff") + if not exe: + sys.exit("odiff not found; install it with `npm install -g odiff-bin` or pass --odiff") + + base_manifest, actual_manifest = read_manifest(baseline), read_manifest(actual) + results = [] + for case in load_cases(): + name = case["name"] + base_png, actual_png, diff_png = ( + baseline / f"{name}.png", + actual / f"{name}.png", + diffs / f"{name}.png", + ) + render_status = actual_manifest["cases"].get(name, {}).get("status", "not rendered") + result = { + "name": name, + "source": case.get("rust") or case.get("python"), + "kind": "rust" if "rust" in case else "python", + "max_diff_percent": case["max_diff_percent"], + "diff_percent": None, + } + if not actual_png.exists(): + result.update(status="error", detail=render_status) + elif not base_png.exists(): + result.update(status="new", detail="no baseline on main") + else: + outcome, percent = odiff(exe, base_png, actual_png, diff_png, case["threshold"]) + result["diff_percent"] = percent + if outcome == "layout": + result.update(status="changed", detail="image size changed") + elif percent > case["max_diff_percent"]: + result.update(status="changed", detail=f"{percent:g}% of pixels differ") + else: + result.update(status="pass", detail="identical" if outcome == "match" else f"{percent:g}% (within tolerance)") + results.append(result) + + report = { + "baseline_commit": base_manifest.get("commit"), + "actual_commit": actual_manifest.get("commit"), + "adapter": actual_manifest.get("adapter"), + "allow_changes": args.allow_changes, + "results": results, + } + (out / "results.json").write_text(json.dumps(report, indent=2)) + (out / "summary.md").write_text(render_summary(report)) + (out / "report.html").write_text(render_html(report, baseline, actual, diffs, actual / "logs")) + + counts = {s: sum(r["status"] == s for r in results) for s in ("pass", "changed", "new", "error")} + print(" ".join(f"{k}={v}" for k, v in counts.items())) + if counts["error"]: + return 1 + if counts["changed"] and not args.allow_changes: + return 1 + return 0 + + +STATUS_ICON = {"pass": "✅", "changed": "❌", "new": "🆕", "error": "💥"} + + +def render_summary(report: dict) -> str: + results = report["results"] + changed = [r for r in results if r["status"] == "changed"] + notable = [r for r in results if r["status"] != "pass"] + total = len(results) + if not notable: + headline = f"**Visual regression: no changes** across {total} cases" + else: + parts = [] + for status, label in (("changed", "changed"), ("error", "failed to render"), ("new", "new")): + n = sum(r["status"] == status for r in results) + if n: + parts.append(f"{n} {label}") + headline = f"**Visual regression: {', '.join(parts)}** of {total} cases" + lines = [headline, ""] + if notable: + lines += ["| case | status | detail |", "|---|---|---|"] + for r in notable: + lines.append(f"| `{r['name']}` | {STATUS_ICON[r['status']]} {r['status']} | {r['detail']} |") + lines.append("") + if changed: + if report["allow_changes"]: + lines.append("Changes accepted by the `deliberate-rendering-change` label.") + else: + lines.append("If these changes are intentional, add the `deliberate-rendering-change` label.") + lines.append("") + base = (report.get("baseline_commit") or "unknown")[:10] + lines.append(f"baseline `{base}` · adapter `{report.get('adapter') or 'unknown'}`") + return "\n".join(lines) + "\n" + + +def data_uri(path: Path) -> str | None: + if not path.exists(): + return None + return "data:image/png;base64," + base64.b64encode(path.read_bytes()).decode() + + +def figure(label: str, path: Path, css_class: str = "") -> str: + uri = data_uri(path) + body = f'{label}' if uri else '
none
' + return f'
{body}
{label}
' + + +def render_html(report: dict, baseline: Path, actual: Path, diffs: Path, logs: Path) -> str: + order = {"error": 0, "changed": 1, "new": 2, "pass": 3} + sections = [] + for r in sorted(report["results"], key=lambda r: (order[r["status"]], r["name"])): + name, status = r["name"], r["status"] + title = ( + f'

{status} {html.escape(name)} ' + f'{html.escape(r["kind"])}: {html.escape(r["source"])} · ' + f'{html.escape(r["detail"])}

' + ) + if status == "pass": + sections.append(f'
{title}
') + continue + figures = "" + if status == "changed": + figures = ( + figure("baseline (main)", baseline / f"{name}.png") + + figure("this PR", actual / f"{name}.png") + + figure("changed pixels", diffs / f"{name}.png", "mask") + ) + elif status == "new": + figures = figure("this PR", actual / f"{name}.png") + log = "" + if status == "error": + log_path = logs / f"{name}.log" + tail = "\n".join(log_path.read_text().splitlines()[-LOG_TAIL_LINES:]) if log_path.exists() else "" + log = f"
{html.escape(tail or 'no log captured')}
" + sections.append(f'
{title}
{figures}
{log}
') + + meta = ( + f"baseline {html.escape(str(report.get('baseline_commit')))} · " + f"PR {html.escape(str(report.get('actual_commit')))} · " + f"adapter {html.escape(str(report.get('adapter')))}" + ) + return f""" + + + + +Visual regression report + + + +
+

Visual regression report

+

{meta}

+
+{"".join(sections)} + + +""" + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + + render = sub.add_parser("render", help="render every case to PNG") + render.add_argument("--out", required=True) + render.add_argument("--only", nargs="+", metavar="NAME") + render.add_argument("--skip-build", action="store_true", help="reuse existing builds") + render.set_defaults(func=cmd_render) + + compare = sub.add_parser("compare", help="diff rendered cases against a baseline") + compare.add_argument("--baseline", required=True) + compare.add_argument("--actual", required=True) + compare.add_argument("--out", required=True) + compare.add_argument("--odiff", help="path to the odiff binary") + compare.add_argument("--allow-changes", action="store_true", help="report changes without failing") + compare.set_defaults(func=cmd_compare) + + args = parser.parse_args() + return args.func(args) + + +if __name__ == "__main__": + sys.exit(main())