diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 8f002e7..de7f404 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -17,24 +17,32 @@ concurrency: cancel-in-progress: true jobs: + # The image of test-e2e-qemu too: one image, one migration. lint-and-test: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 60 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + # gcc's 32-bit libraries, which test-i686 links with. + - run: | + sudo apt-get update + sudo apt-get install --yes gcc-multilib + - run: make setup # After setup: the key covers installed toolchains. - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 # Every check but the boots. - - run: make -k lint test + - run: make -k lint test test-i686 + # QEMU 9.0 and newer fault unaligned loads from Device memory as arm64 does, which the + # aarch64 guests check; Ubuntu 24.04 has QEMU 8.2. test-e2e-qemu: - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 timeout-minutes: 120 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -57,23 +65,24 @@ jobs: # profile and logs errors; the builds take about a minute without it. - run: make setup - # Kernel RPMs and DriverSample.efi. Saved only on success, as run.sh reuses - # DriverSample.efi unchecked. + # The guests' packages and each architecture's DriverSample.efi, keyed also on the + # apt packages and the runner image (this file) that build DriverSample. Saved only + # on success, as run.sh reuses DriverSample.efi unchecked. - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: | - target/qemu-test/kernel-rpms - target/qemu-test/driver-sample - key: qemu-inputs-${{ hashFiles('tests/qemu/kernel.sha256', 'tests/qemu/build-driver-sample.sh') }} + target/qemu-test/packages + target/qemu-test/*/driver-sample + key: qemu-inputs-${{ hashFiles('tests/qemu/guest-packages-*', 'tests/qemu/build-driver-sample.sh', 'tests/qemu/ubuntu-packages', '.github/workflows/checks.yml') }} - - run: make test-e2e-qemu + - run: make -k test-e2e-qemu - if: failure() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: qemu-test-logs path: | - target/qemu-test/*/*/*.log - target/qemu-test/*/executables + target/qemu-test/*/*/*/*.log + target/qemu-test/*/*/executables if-no-files-found: ignore retention-days: 14 diff --git a/Cargo.lock b/Cargo.lock index 2d183ff..153a62b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -171,6 +171,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + [[package]] name = "either" version = "1.9.0" @@ -1047,6 +1056,7 @@ dependencies = [ name = "uefisettings-qemu-guest" version = "0.1.0" dependencies = [ + "crc32fast", "serde", "serde_json", ] diff --git a/Makefile b/Makefile index bb4c5c1..012a031 100644 --- a/Makefile +++ b/Makefile @@ -11,13 +11,15 @@ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # The checks of uefisettings. They need GNU make and bash, rustup while -# RUST_TOOLCHAIN is set, as it is by default, and for test-e2e-qemu an x86_64 -# host with /dev/kvm and what tests/qemu/README.md lists: +# RUST_TOOLCHAIN is set, as it is by default, for test-i686 an x86 host with +# gcc's 32-bit libraries, which README.md names, and for test-e2e-qemu an +# x86_64 host with /dev/kvm and what tests/qemu/README.md lists: # make every check of CHECKS, stopping at the first that fails # make -k every check of CHECKS, even after one fails # make ... the named checks, e.g. make clippy-aarch64 doc # make lint the checks of LINTS -# make lint test every check but test-e2e-qemu +# make lint test test-i686 +# every check but test-e2e-qemu # make RUST_TOOLCHAIN= the checks on the cargo of PATH # make setup installs, through rustup, the toolchain, components and # targets that the checks use @@ -27,10 +29,10 @@ # # A CI host runs make setup && make -k under timeout, as nothing bounds the time # that cargo takes: timeout -k 60 bash -c 'make setup && make -k'. It -# may keep target/qemu-test/kernel-rpms and target/qemu-test/driver-sample -# between runs: tests/qemu/fetch-kernel.sh downloads only the RPMs that are +# may keep target/qemu-test/packages and target/qemu-test/*/driver-sample +# between runs: tests/qemu/fetch-packages.sh downloads only the packages that are # missing there or fail their sha256 check, and tests/qemu/run.sh builds -# DriverSample only when it is missing, so delete target/qemu-test/driver-sample +# DriverSample only when it is missing, so delete target/qemu-test/*/driver-sample # after a change to build-driver-sample.sh. SHELL := bash @@ -58,14 +60,16 @@ export RUSTFLAGS := -D warnings unexport CARGO_ENCODED_RUSTFLAGS # The targets that the checks build for, comma-separated as rustup takes them. -TARGETS := x86_64-unknown-linux-gnu,aarch64-unknown-linux-gnu,x86_64-unknown-linux-musl,x86_64-unknown-uefi +TARGETS := x86_64-unknown-linux-gnu,aarch64-unknown-linux-gnu,i686-unknown-linux-gnu,x86_64-unknown-linux-musl,x86_64-unknown-uefi,i686-unknown-linux-musl,i686-unknown-uefi,aarch64-unknown-linux-musl,aarch64-unknown-uefi comma := , -# CI runs lint test and test-e2e-qemu: a new check joins LINTS or gets a CI job. -LINTS := fmt clippy-x86_64 clippy-aarch64 clippy-uefi doc -CHECKS := $(LINTS) test test-e2e-qemu +# CI runs lint test test-i686 and test-e2e-qemu: a new check joins LINTS or +# QEMU_TESTS or CI runs it too. +LINTS := fmt clippy-x86_64 clippy-aarch64 clippy-i686 clippy-uefi doc +QEMU_TESTS := test-e2e-qemu-x86_64 test-e2e-qemu-i686 test-e2e-qemu-aarch64 +CHECKS := $(LINTS) test test-i686 test-e2e-qemu -.PHONY: check lint help setup toolchain $(CHECKS) +.PHONY: check lint help setup toolchain $(CHECKS) $(QEMU_TESTS) check: $(CHECKS) @@ -75,7 +79,8 @@ help: @echo "checks: $(CHECKS)" @echo "make runs every check; make ... runs the named ones." @echo "make lint runs $(LINTS)." - @echo "make lint test runs every check but test-e2e-qemu." + @echo "make test-e2e-qemu runs $(QEMU_TESTS)." + @echo "make lint test test-i686 runs every check but test-e2e-qemu." @echo "make -k runs every check, even after one fails." @echo "make -n prints the commands of a check without running them." @echo "make RUST_TOOLCHAIN= runs the checks on the cargo of PATH." @@ -101,7 +106,7 @@ toolchain: fmt: toolchain cargo fmt --check -clippy-x86_64 clippy-aarch64: clippy-%: toolchain +clippy-x86_64 clippy-aarch64 clippy-i686: clippy-%: toolchain cargo clippy --locked --workspace --all-targets --all-features --target $*-unknown-linux-gnu -- -D warnings # publish-hiidb, the one crate built for UEFI. @@ -111,10 +116,17 @@ clippy-uefi: toolchain test: toolchain scripts/assert-tests-ran.sh cargo test --locked +# The tests on 32-bit glibc, where off_t is signed and has 32 bits; this +# host runs them. +test-i686: toolchain + scripts/assert-tests-ran.sh cargo test --locked --target i686-unknown-linux-gnu + doc: toolchain RUSTDOCFLAGS='-D warnings' cargo doc --workspace --no-deps --document-private-items --locked -# Boots the scenarios of tests/qemu/scenarios.tsv under KVM, with logs in -# target/qemu-test//; see tests/qemu/README.md. -test-e2e-qemu: toolchain - tests/qemu/run.sh +test-e2e-qemu: $(QEMU_TESTS) + +# Boots the scenarios of tests/qemu/scenarios.tsv on one architecture, with logs in +# target/qemu-test///; see tests/qemu/README.md. +$(QEMU_TESTS): test-e2e-qemu-%: toolchain + tests/qemu/run.sh $* diff --git a/README.md b/README.md index 0358088..fa7fb41 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ mv lib.rs thrift/rust/uefisettings_spellings_db_thrift/uefisettings_spellings_db ## Checks -The `Makefile` in the root of the repository runs the checks that a change should pass: the lints (`make lint`: rustfmt, clippy for x86_64, aarch64 and UEFI, and the docs), the tests (`make test`) and the QEMU tests (`make test-e2e-qemu`). +The `Makefile` in the root of the repository runs the checks that a change should pass: the lints (`make lint`: rustfmt, clippy for x86_64, aarch64, i686 and UEFI, and the docs), the tests (`make test`, and `make test-i686` on 32-bit glibc) and the QEMU tests (`make test-e2e-qemu`). The checks need GNU make and bash, and rustup unless `make RUST_TOOLCHAIN=` runs them on the cargo of `PATH`. @@ -153,11 +153,13 @@ make setup # once: installs, through rustup, the Rust toolchain, compo make # every check, stopping at the first that fails make -k # every check, even after one fails make ... # the named checks, e.g. make clippy-aarch64 doc -make lint test # every check but the QEMU tests +make lint test test-i686 # every check but the QEMU tests make RUST_TOOLCHAIN= # the checks on the cargo of PATH make help # lists the checks ``` +`make test-i686` also needs an x86 host (x86_64 or i686), which runs the i686 test binaries, and gcc's 32-bit libraries (Ubuntu: `gcc-multilib`, or, next to the `gcc-aarch64-linux-gnu` of [`tests/qemu/ubuntu-packages`](tests/qemu/ubuntu-packages), which conflicts with it, `libc6-dev-i386` and `lib32gcc--dev`; Fedora: `glibc-devel.i686`). + `make test-e2e-qemu`, the QEMU tests, also needs an x86_64 host with `/dev/kvm` and the packages that [`tests/qemu/README.md`](tests/qemu/README.md) lists. GitHub Actions runs the checks on every pull request and every push to main, as [`.github/workflows/checks.yml`](.github/workflows/checks.yml) describes. diff --git a/src/lib/chattr.rs b/src/lib/chattr.rs index 6e89405..a8d6119 100644 --- a/src/lib/chattr.rs +++ b/src/lib/chattr.rs @@ -11,6 +11,8 @@ // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. use std::fs::OpenOptions; +use std::os::raw::c_long; +use std::os::raw::c_uint; use std::os::unix::io::AsRawFd; use std::path::Path; use std::path::PathBuf; @@ -26,24 +28,24 @@ const FS_IOC_GETFLAGS: u8 = 1; const FS_IOC_SETFLAGS: u8 = 2; // full list of flags https://github.com/torvalds/linux/blob/master/include/uapi/linux/fs.h#L239 -pub const FS_IMMUTABLE_FL: i64 = 0x00000010; // Immutable file +pub const FS_IMMUTABLE_FL: c_uint = 0x00000010; // Immutable file -nix::ioctl_read!( +// The requests encode the size of a long, but the kernel reads and writes an unsigned +// int (ioctl_getflags() and ioctl_setflags() in fs/file_attr.c, Linux v6.17). +nix::ioctl_read_bad!( ioctl_fs_get_attrs, - FS_IOC_MAGIC, - FS_IOC_GETFLAGS, - ::std::os::raw::c_long + nix::request_code_read!(FS_IOC_MAGIC, FS_IOC_GETFLAGS, std::mem::size_of::()), + c_uint ); -nix::ioctl_write_ptr!( +nix::ioctl_write_ptr_bad!( ioctl_fs_set_attrs, - FS_IOC_MAGIC, - FS_IOC_SETFLAGS, - ::std::os::raw::c_long + nix::request_code_write!(FS_IOC_MAGIC, FS_IOC_SETFLAGS, std::mem::size_of::()), + c_uint ); /// returns all extended fs attributes for the given path as a set of bit flags -pub fn get_all_attrs(path: impl AsRef) -> Result { +pub fn get_all_attrs(path: impl AsRef) -> Result { let path = path.as_ref(); let file = OpenOptions::new() .write(false) @@ -54,13 +56,11 @@ pub fn get_all_attrs(path: impl AsRef) -> Result { path.to_string_lossy() ))?; let fd = file.as_raw_fd(); - let mut attrs = 0i64; - // SAFETY c_long is guaranteed to be i32 or i64 (https://doc.rust-lang.org/std/os/raw/type.c_long.html) - // This tool is linux specific and do not support 32bit systems, so we can assume that it's always i64 - let attrs_ptr = &mut attrs as *mut i64; - + let mut attrs: c_uint = 0; + // SAFETY: fd stays open while file lives, and attrs is the unsigned int that the + // kernel writes. unsafe { - ioctl_fs_get_attrs(fd, attrs_ptr).context("ioctl failed")?; + ioctl_fs_get_attrs(fd, &mut attrs).context("ioctl failed")?; } Ok(attrs) @@ -68,7 +68,7 @@ pub fn get_all_attrs(path: impl AsRef) -> Result { /// writes extended fs attributes for the given path as a set of bit flags /// all existing attributes will be overwritten -pub fn set_all_attrs(path: impl AsRef, attrs: i64) -> Result<()> { +pub fn set_all_attrs(path: impl AsRef, attrs: c_uint) -> Result<()> { let path = path.as_ref(); let file = OpenOptions::new() .write(false) @@ -79,12 +79,10 @@ pub fn set_all_attrs(path: impl AsRef, attrs: i64) -> Result<()> { path.to_string_lossy() ))?; let fd = file.as_raw_fd(); - let mut tmp = attrs; - // SAFETY c_long is guaranteed to be i32 or i64 (https://doc.rust-lang.org/std/os/raw/type.c_long.html) - // This tool is linux specific and do not support 32bit systems, so we can assume that it's always i64 - let attrs_ptr = &mut tmp as *mut i64; + // SAFETY: fd stays open while file lives, and attrs is the unsigned int that the + // kernel reads. unsafe { - ioctl_fs_set_attrs(fd, attrs_ptr).context("ioctl failed")?; + ioctl_fs_set_attrs(fd, &attrs).context("ioctl failed")?; } Ok(()) } @@ -127,7 +125,7 @@ macro_rules! clear_attrs { pub struct EfivarsImmutabilityGuard { path: PathBuf, - attrs_before_writing: i64, + attrs_before_writing: c_uint, } impl EfivarsImmutabilityGuard { @@ -180,7 +178,7 @@ pub mod tests { use super::*; //in the tests we expect that FS_NODUMP_FL is initially cleared, might be a weak assumption.. - pub const FS_NODUMP_FL: i64 = 0x00000040; // do not dump file + pub const FS_NODUMP_FL: c_uint = 0x00000040; // do not dump file #[test] fn test_get_all_attrs() -> Result<()> { let test_file = tempfile::NamedTempFile::new().expect("Failed to create test file"); @@ -230,6 +228,14 @@ pub mod tests { get_all_attrs(test_file.path()).context("No error expected from get_attrs")?; // checking that FS_NODUMP_FL is set assert_eq!(attrs_after_set & FS_NODUMP_FL, FS_NODUMP_FL); + // checking that clearing a flag that no new file has or inherits, so that clearing it + // needs no capability, keeps FS_NODUMP_FL, the only flag that the owner may set on both + // ZFS and tmpfs + clear_attrs!(test_file.path(), FS_IMMUTABLE_FL) + .context("No error expected from clear_attrs")?; + let attrs_after_other_clear = + get_all_attrs(test_file.path()).context("No error expected from get_attrs")?; + assert_eq!(attrs_after_other_clear & FS_NODUMP_FL, FS_NODUMP_FL); clear_attrs!(test_file.path(), FS_NODUMP_FL, FS_NODUMP_FL) .context("No error expected from clear_attrs")?; let attrs_after_clear = @@ -237,7 +243,7 @@ pub mod tests { // checking that FS_NODUMP_FL is cleared assert_eq!(attrs_after_clear & FS_NODUMP_FL, 0); // checking that everything else is still the same - assert_eq!(attrs_after_set & !FS_NODUMP_FL, attrs & !FS_NODUMP_FL); + assert_eq!(attrs_after_clear & !FS_NODUMP_FL, attrs & !FS_NODUMP_FL); Ok(()) } } diff --git a/src/lib/hii/extract.rs b/src/lib/hii/extract.rs index 48d50ae..2a1ed23 100644 --- a/src/lib/hii/extract.rs +++ b/src/lib/hii/extract.rs @@ -197,13 +197,14 @@ fn copy_mapped_page(file: &File, offset: u64, dst: &mut [u8]) -> Result { let offset_in_page = usize::try_from(offset % page_size as u64)?; let len = (page_size - offset_in_page).min(dst.len()); let page_base = offset - offset_in_page as u64; - let map_base = libc::off_t::try_from(page_base) - .with_context(|| format!("Mapping offset 0x{page_base:x} does not fit off_t"))?; + // mmap64(), as off_t is signed and has 32 bits on most 32-bit glibc targets. + let map_base = libc::off64_t::try_from(page_base) + .with_context(|| format!("Mapping offset 0x{page_base:x} does not fit off64_t"))?; // SAFETY: addr = NULL lets the kernel pick an unused address range, so the // new PROT_READ mapping cannot alias any Rust allocation (including dst). let mapped = unsafe { - libc::mmap( + libc::mmap64( ptr::null_mut(), page_size, libc::PROT_READ, @@ -356,7 +357,7 @@ mod tests { } } - /// The offset also fails the off_t check; the overflow error must win. + /// The offset also fails the off64_t check; the overflow error must win. #[test] fn read_mapped_rejects_overflowing_range() { let (file, _) = mapped_fixture(16); diff --git a/tests/qemu/README.md b/tests/qemu/README.md index aa90607..620350f 100644 --- a/tests/qemu/README.md +++ b/tests/qemu/README.md @@ -1,12 +1,14 @@ # QEMU tests -These tests boot QEMU guests under OVMF and run `uefisettings` inside them, so they +These tests boot x86_64, i686 and aarch64 QEMU guests under EDK2's firmware (OVMF, or +AAVMF on aarch64) and run `uefisettings` inside them, so they exercise the CLI against real UEFI firmware, efivarfs and `/dev/mem` without touching the machine that runs them. ## Running -`make test-e2e-qemu` runs every scenario, on the Rust +`make test-e2e-qemu` runs every scenario on every architecture, and +`make test-e2e-qemu-` on one of `x86_64`, `i686` and `aarch64`, on the Rust toolchain and targets that `make setup` installs (see the Checks section of the repository's README): @@ -15,36 +17,47 @@ make setup # once make test-e2e-qemu ``` -It needs an x86_64 host with `/dev/kvm` and, on Ubuntu 24.04, the packages of +It needs an x86_64 host with `/dev/kvm`, which the x86_64 and i686 guests use; QEMU +emulates the aarch64 guests, and must be 9.0 or newer, which faults unaligned loads from +Device memory as arm64 does. On Ubuntu 26.04 it also needs the packages of [`ubuntu-packages`](ubuntu-packages): ```sh xargs --arg-file=tests/qemu/ubuntu-packages sudo apt-get install --yes ``` -`run.sh` alone runs the scenarios on the active toolchain of rustup, which then needs the -musl and UEFI targets: +The guest's kernel, firmware and busybox do not come from the host: they are the packages +that `guest-packages-` ([x86_64](guest-packages-x86_64), [i686](guest-packages-i686) +or [aarch64](guest-packages-aarch64)) pins, which +[`fetch-packages.sh`](fetch-packages.sh) downloads. + +`run.sh` alone runs the scenarios of an architecture on the active toolchain of rustup, +which then needs that architecture's musl and UEFI targets: ```sh rustup target add x86_64-unknown-linux-musl x86_64-unknown-uefi # once -tests/qemu/run.sh # every scenario -tests/qemu/run.sh boot-smoke # the named scenarios +tests/qemu/run.sh x86_64 # every scenario +tests/qemu/run.sh x86_64 boot-smoke # the named scenarios ``` -Elsewhere, the environment variables in the header of [`run.sh`](run.sh) give the paths -of QEMU, the OVMF code and variable images, a static busybox and the directory of -DriverSample.efi, and these tools must be on `PATH`: `virt-fw-vars`, `jq` and `cpio` for -`run.sh`; `curl`, `rpm2cpio` and `cpio` for [`fetch-kernel.sh`](fetch-kernel.sh); and, +Elsewhere, these tools must be on `PATH`: `qemu-system-x86_64`, `qemu-system-i386`, +`qemu-system-aarch64`, `virt-fw-vars`, `jq` and `cpio` for `run.sh`; `curl`, `rpm2cpio`, +`cpio`, `ar`, `tar`, `xz` and `zstd` for `fetch-packages.sh`; and, unless DriverSample.efi is already built, `gcc`, `g++`, `make`, `git`, `nasm`, `python3` and the libuuid headers for [`build-driver-sample.sh`](build-driver-sample.sh). +The aarch64 builds also need `aarch64-linux-gnu-gcc` with the `libdl.a` of an aarch64 +glibc, or another linker that `CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER` names, and +the i686 build of `uefisettings` a 32-bit `libdl.a` (Fedora: `glibc-devel.i686`), as the +`libloading` crate links `-ldl`. -`QEMU` must name the `qemu-system-x86_64` binary or a wrapper that `exec`s it: `run.sh` +The `qemu-system-*` on `PATH` may be a wrapper that `exec`s QEMU: `run.sh` stops a boot after 120 s by signalling that program, not its children, so a wrapper that runs QEMU as a child would leave the guest running. -Each run writes to `target/qemu-test//`, with one directory per scenario. For -each boot `N`, `bootN.console.log` has the firmware, kernel and `init` output and -`bootN.test.log` the test output. +Each run writes to `target/qemu-test///`, with one directory per +scenario. For each boot `N`, `bootN.console.log` has the firmware, kernel and `init` +output, `bootN.test.log` the test output and `bootN.qemu.log` how QEMU saw the guest end +(`run.sh` explains its lines). [`scenarios.tsv`](scenarios.tsv) lists the scenarios and the boots of each; each file in [`guest/tests/`](guest/tests/) is one test binary, which a boot names, and its module @@ -55,7 +68,7 @@ command line lacks `uefisettings.test=`. They build only with the `test-vm` feat ## HII fixture -OVMF does not publish its HII database for the OS, so the HII scenarios have the firmware +EDK2's firmware does not publish its HII database for the OS, so the HII scenarios have it run two drivers from the ESP before the kernel, through `Driver####` load options: - `DriverSample.efi`, EDK2's unmodified sample HII driver (BSD-2-Clause-Patent), which @@ -63,6 +76,7 @@ run two drivers from the ESP before the kernel, through `Driver####` load option the first run; - `publish-hiidb.efi`, from [`publish-hiidb/`](publish-hiidb/), which points the `HiiDB` variable at a copy of the HII database, the way the firmware of OCP machines does. + It also puts the CRC-32 of the database in the `HiiDBCrc32` variable for the tests. [`vars/my-ifr-nv-data.json`](vars/my-ifr-nv-data.json) creates DriverSample's variable first, with runtime access, so that efivarfs shows it, and with DriverSample's defaults at diff --git a/tests/qemu/build-driver-sample.sh b/tests/qemu/build-driver-sample.sh index 45b53c8..26502f3 100755 --- a/tests/qemu/build-driver-sample.sh +++ b/tests/qemu/build-driver-sample.sh @@ -12,25 +12,34 @@ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Builds DriverSample.efi, EDK2's sample HII driver -# (MdeModulePkg/Universal/DriverSampleDxe), which the QEMU tests load into OVMF -# for its HII forms. The source is the edk2-stable202508 tag, checked against +# (MdeModulePkg/Universal/DriverSampleDxe), which the QEMU tests load into EDK2's firmware +# for its HII forms. The source is the edk2-stable202608 tag, checked against # its commit. DriverSample is BSD-2-Clause-Patent, so EDK2's License.txt is # copied next to it as DriverSample-License.txt. # Needs gcc, g++, make, nasm, python3, git and the libuuid headers; on Ubuntu # 24.04: apt-get install build-essential git nasm python3 uuid-dev. +# AARCH64 also needs the gcc that GCC_AARCH64_PREFIX names (default: aarch64-linux-gnu-, +# which gcc-aarch64-linux-gnu has). # JOBS sets the make and build job count (default: nproc). # -# Usage: [JOBS=] tests/qemu/build-driver-sample.sh +# Usage: [JOBS=] tests/qemu/build-driver-sample.sh set -euo pipefail -if (($# != 1)); then - echo "usage: $0 " >&2 +arch=${1-} +case $arch in +X64 | IA32 | AARCH64) ;; +*) arch= ;; +esac +if (($# != 2)) || [[ -z $arch ]]; then + echo "usage: $0 " >&2 exit 2 fi -out=$(realpath -m -- "$1") -tag=edk2-stable202508 -commit=d46aa46c8361194521391aa581593e556c707c6e +out=$(realpath -m -- "$2") +# EDK2's tools_def.txt reads the prefix of the AARCH64 gcc from it. +export GCC_AARCH64_PREFIX=${GCC_AARCH64_PREFIX-aarch64-linux-gnu-} +tag=edk2-stable202608 +commit=2970e5699ba6267f3384ffab20f96647578aebc8 jobs=${JOBS:-$(nproc)} work=$(mktemp -d) @@ -54,8 +63,17 @@ set -- && set +u # shellcheck source=/dev/null . ./edksetup.sh set -u -build -a X64 -t GCC -b RELEASE -n "$jobs" -p MdeModulePkg/MdeModulePkg.dsc \ +# EDK2's X64 flags name no -march, so gcc's default applies, and that varies: Ubuntu's +# amd64v3 gcc emits AVX, which faults in the firmware (CR4.OSXSAVE is clear). The DSC's +# build options append -march=x86-64, the baseline, to EDK2's flags. +cat >>MdeModulePkg/MdeModulePkg.dsc <<'EOF' + +[BuildOptions] + GCC:*_*_X64_CC_FLAGS = -march=x86-64 + GCC:*_*_X64_DLINK_FLAGS = -march=x86-64 +EOF +build -a "$arch" -t GCC -b RELEASE -n "$jobs" -p MdeModulePkg/MdeModulePkg.dsc \ -m MdeModulePkg/Universal/DriverSampleDxe/DriverSampleDxe.inf mkdir -p -- "$out" -cp Build/MdeModule/RELEASE_GCC/X64/DriverSample.efi "$out/" +cp "Build/MdeModule/RELEASE_GCC/$arch/DriverSample.efi" "$out/" cp License.txt "$out/DriverSample-License.txt" diff --git a/tests/qemu/fetch-kernel.sh b/tests/qemu/fetch-kernel.sh deleted file mode 100755 index 732fb55..0000000 --- a/tests/qemu/fetch-kernel.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -# Copyright 2023 Meta Platforms, Inc. and affiliates. -# -# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: -# -# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -# -# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -# -# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# Downloads the guest kernel's RPMs that tests/qemu/kernel.sha256 lists into CACHE, checks -# them against their sha256 and unpacks them into the new directory ROOT. RPMs already in -# CACHE with the right sha256 are not downloaded again. Each mirror gets at most 1220 s for -# an RPM, so the download of an RPM from both mirrors stops within about 41 minutes. -# -# Usage: tests/qemu/fetch-kernel.sh CACHE ROOT - -set -euo pipefail - -if (($# != 2)); then - echo "usage: $0 CACHE ROOT" >&2 - exit 2 -fi -cache=$1 -root=$2 -list=$(realpath "$(dirname "$0")/kernel.sha256") -# Fedora 43 packages; the archive keeps them once the release reaches its end of life. -mirrors=( - https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os/Packages/k - https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/43/Everything/x86_64/os/Packages/k -) - -mkdir -p "$cache" -cd "$cache" -while read -r sum rpm; do - if ! sha256sum --check --status <<<"$sum $rpm" 2>/dev/null; then - # --max-time ends a try that stalls: kernel-core, about 21 MB, takes 300 s only - # below 70 kB/s. curl makes up to 4 tries with waits of 1, 2 and 4 s between - # them, 1207 s in all, but waits longer when the server sends a Retry-After - # header, so timeout stops curl after 1210 s and kills it 10 s later. Without - # --foreground, timeout would move curl into a process group of its own, which a - # timeout around make does not reach. - for mirror in "${mirrors[@]}"; do - timeout --foreground -k 10 1210 curl --fail --silent --show-error --location \ - --retry 3 --max-time 300 --output "$rpm" "$mirror/$rpm" && break - done - fi -done <"$list" -sha256sum --check --strict "$list" - -mkdir "$root" -while read -r _ rpm; do - rpm2cpio "$rpm" | (cd "$root" && cpio --extract --make-directories --preserve-modification-time --quiet) -done <"$list" diff --git a/tests/qemu/fetch-packages.sh b/tests/qemu/fetch-packages.sh new file mode 100755 index 0000000..5ec55d0 --- /dev/null +++ b/tests/qemu/fetch-packages.sh @@ -0,0 +1,136 @@ +#!/bin/bash +# Copyright 2023 Meta Platforms, Inc. and affiliates. +# +# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +# +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Downloads the packages that LIST names into CACHE, checks them against their sha256 and +# unpacks them, .rpm and .deb alike, into the new directory ROOT. Each line of LIST that is +# neither empty nor a # comment holds the sha256 of a package and then its HTTPS URLs, +# mirrors of one another that end in the same file name, with a permanent archive last, as +# distributions drop superseded packages from their pools. The whole list is checked before +# any download. A package already in CACHE with the right sha256 is not downloaded again; +# otherwise its URLs are tried in order, the next one after a failed download or a sha256 +# mismatch, each for at most 1220 s. ROOT appears only once every package is right and +# unpacked. +# +# Usage: tests/qemu/fetch-packages.sh LIST CACHE ROOT + +set -euo pipefail + +if (($# != 3)); then + echo "usage: $0 LIST CACHE ROOT" >&2 + exit 2 +fi +list=$(realpath "$1") +cache=$2 +root=$(realpath -m "$3") +if [[ -e $root ]]; then + echo "$root exists" >&2 + exit 1 +fi + +sums=() +files=() +mirrors=() +# read fails on a last line without a newline, which still counts. +while read -r sum urls || [[ -n $sum ]]; do + [[ -z $sum || $sum == \#* ]] && continue + if [[ ! $sum =~ ^[0-9a-f]{64}$ ]]; then + echo "$list: $sum is not a sha256" >&2 + exit 1 + fi + read -r -a urls <<<"$urls" + if ((${#urls[@]} == 0)); then + echo "$list: no URL for $sum" >&2 + exit 1 + fi + file=${urls[0]##*/} + case $file in + *.rpm | *.deb) ;; + *) + echo "$list: $file is neither an .rpm nor a .deb" >&2 + exit 1 + ;; + esac + for url in "${urls[@]}"; do + if [[ $url != https://* ]]; then + echo "$list: $url is not an HTTPS URL" >&2 + exit 1 + fi + if [[ ${url##*/} != "$file" ]]; then + echo "$list: the URLs of $file name other files: ${urls[*]}" >&2 + exit 1 + fi + done + sums+=("$sum") + files+=("$file") + mirrors+=("${urls[*]}") +done <"$list" +if ((${#files[@]} == 0)); then + echo "$list: no package" >&2 + exit 1 +fi + +mkdir -p "$cache" +cd "$cache" +for i in "${!files[@]}"; do + sum=${sums[i]} + file=${files[i]} + read -r -a urls <<<"${mirrors[i]}" + # The -f test keeps the sha256sum of uutils (Ubuntu 26.04) from printing "FAILED open + # or read" for a missing file, which it does even with --status. + if ! { [[ -f $file ]] && sha256sum --check --status <<<"$sum $file"; } 2>/dev/null; then + # --max-time ends a try that stalls: the i686 kernel, the biggest package at about + # 52 MB, takes 300 s only below 175 kB/s. curl makes up to 4 tries with waits of 1, + # 2 and 4 s between them, 1207 s in all, but waits longer when the server sends a + # Retry-After header, so timeout stops curl after 1210 s and kills it 10 s later. + # Without --foreground, timeout would move curl into a process group of its own, + # which a timeout around make does not reach. + for url in "${urls[@]}"; do + timeout --foreground -k 10 1210 curl --fail --silent --show-error --location \ + --proto =https --proto-redir =https --retry 3 --max-time 300 \ + --output "$file" "$url" && + sha256sum --check --status <<<"$sum $file" && break + done + fi + sha256sum --check --strict <<<"$sum $file" +done + +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +# Unpacked next to ROOT and renamed to it at the end, so that a failure leaves no ROOT. +staging=$(mktemp -d "$root.XXXXXX") +trap 'rm -rf "$work" "$staging"' EXIT +# mkdir's mode, not mktemp's 0700. +chmod "$(umask -S)" "$staging" +unpack_failed() { + echo "fetch-packages.sh: cannot unpack $1" >&2 + exit 1 +} +for file in "${files[@]}"; do + echo "fetch-packages.sh: unpacking $file" + case $file in + *.rpm) + rpm2cpio "$file" | + (cd "$staging" && cpio --extract --make-directories --preserve-modification-time \ + --no-absolute-filenames --quiet) || + unpack_failed "$file" + ;; + *.deb) + # An ar archive whose member data.tar, compressed or not, holds the files. + data=$(ar t "$file" | grep -E --line-regexp 'data\.tar(\.[a-z0-9]+)?') || + unpack_failed "$file" + ar p "$file" "$data" >"$work/$data" || unpack_failed "$file" + tar --extract --file "$work/$data" --directory "$staging" || unpack_failed "$file" + ;; + esac +done +mv -T "$staging" "$root" diff --git a/tests/qemu/guest-packages-aarch64 b/tests/qemu/guest-packages-aarch64 new file mode 100644 index 0000000..3f56f19 --- /dev/null +++ b/tests/qemu/guest-packages-aarch64 @@ -0,0 +1,4 @@ +# The kernel, firmware and busybox of the aarch64 guests, in the format of fetch-packages.sh. +9328e61443dcd71e5671806fbef4345eb15a988229b011340fac8b856315c3ee https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/aarch64/os/Packages/k/kernel-core-6.17.1-300.fc43.aarch64.rpm https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/43/Everything/aarch64/os/Packages/k/kernel-core-6.17.1-300.fc43.aarch64.rpm +fad8e80e929cd00dae8c5c8e5dd3ec85f37d9a832eea776957e5e2cf86ef2ab5 https://archive.ubuntu.com/ubuntu/pool/main/e/edk2/qemu-efi-aarch64_2024.02-2ubuntu0.10_all.deb https://launchpad.net/ubuntu/+archive/primary/+files/qemu-efi-aarch64_2024.02-2ubuntu0.10_all.deb +d96535e0402c011e0ee43449799df2f4504d44b842e4f2b3a6cbc845508eaafc https://ports.ubuntu.com/ubuntu-ports/pool/main/b/busybox/busybox-static_1.36.1-6ubuntu3.1_arm64.deb https://launchpad.net/ubuntu/+archive/primary/+files/busybox-static_1.36.1-6ubuntu3.1_arm64.deb diff --git a/tests/qemu/guest-packages-i686 b/tests/qemu/guest-packages-i686 new file mode 100644 index 0000000..cbab1a6 --- /dev/null +++ b/tests/qemu/guest-packages-i686 @@ -0,0 +1,4 @@ +# The kernel, firmware and busybox of the i686 guests, in the format of fetch-packages.sh. +0d6241268f9344a998c3339c11e527ea8048d409d6c770f67c573707606a3f2e https://deb.debian.org/debian/pool/main/l/linux-signed-i386/linux-image-6.1.0-50-686-pae_6.1.176-1_i386.deb https://snapshot.debian.org/archive/debian/20260704T211317Z/pool/main/l/linux-signed-i386/linux-image-6.1.0-50-686-pae_6.1.176-1_i386.deb +45fb466babfad1f5a51c65d656ba338ae46a7b8bbcda61234b79753a7359d423 https://archive.ubuntu.com/ubuntu/pool/universe/e/edk2/ovmf-ia32_2024.02-2ubuntu0.10_all.deb https://launchpad.net/ubuntu/+archive/primary/+files/ovmf-ia32_2024.02-2ubuntu0.10_all.deb +39c410d5bc570ce2a4e14055bcf021afbae75bc0efe850634138234828b3416a https://archive.ubuntu.com/ubuntu/pool/main/b/busybox/busybox-static_1.36.1-6ubuntu3.1_i386.deb https://launchpad.net/ubuntu/+archive/primary/+files/busybox-static_1.36.1-6ubuntu3.1_i386.deb diff --git a/tests/qemu/guest-packages-x86_64 b/tests/qemu/guest-packages-x86_64 new file mode 100644 index 0000000..cca7ea8 --- /dev/null +++ b/tests/qemu/guest-packages-x86_64 @@ -0,0 +1,4 @@ +# The kernel, firmware and busybox of the x86_64 guests, in the format of fetch-packages.sh. +ab1418c266e2b4b26090ab13e96c249a6bbe7a6fadca09c4e98c9ff477a5d58f https://dl.fedoraproject.org/pub/fedora/linux/releases/43/Everything/x86_64/os/Packages/k/kernel-core-6.17.1-300.fc43.x86_64.rpm https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/43/Everything/x86_64/os/Packages/k/kernel-core-6.17.1-300.fc43.x86_64.rpm +78a6a3a4ffa666f1cb25e9069e4ec9270b5ceac7987b6eefc2674bd3f58a1473 https://archive.ubuntu.com/ubuntu/pool/main/e/edk2/ovmf_2024.02-2ubuntu0.10_all.deb https://launchpad.net/ubuntu/+archive/primary/+files/ovmf_2024.02-2ubuntu0.10_all.deb +944b2728f53ceb3916cec2c962873c9951e612408099601751db2a0a5d81e0ed https://archive.ubuntu.com/ubuntu/pool/main/b/busybox/busybox-static_1.36.1-6ubuntu3.1_amd64.deb https://launchpad.net/ubuntu/+archive/primary/+files/busybox-static_1.36.1-6ubuntu3.1_amd64.deb diff --git a/tests/qemu/guest/Cargo.toml b/tests/qemu/guest/Cargo.toml index 17a83e9..5335e2b 100644 --- a/tests/qemu/guest/Cargo.toml +++ b/tests/qemu/guest/Cargo.toml @@ -10,6 +10,9 @@ description = "Tests of uefisettings that run inside the QEMU test guests (tests serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +[dev-dependencies] +crc32fast = "1.4" + # The tests run only in the QEMU test guests, so they build only with this feature, which # tests/qemu/run.sh enables, and `cargo test --workspace` elsewhere builds no test binary # of this crate. Each file in tests/ needs a [[test]] entry that requires the feature. diff --git a/tests/qemu/guest/src/lib.rs b/tests/qemu/guest/src/lib.rs index 4a1a4ae..d431a13 100644 --- a/tests/qemu/guest/src/lib.rs +++ b/tests/qemu/guest/src/lib.rs @@ -13,10 +13,10 @@ //! Helpers of the tests that run inside the QEMU test guests. //! //! `tests/qemu/run.sh` builds this crate's test binaries for musl and boots each scenario -//! under OVMF, with the binaries and a static `uefisettings` in the initramfs. +//! under EDK2's firmware, with the binaries and a static `uefisettings` in the initramfs. //! `tests/qemu/init` runs the test binary that the boot's kernel command line names, then //! checks the kernel's health. The tests assert inside the guest; libtest reports their -//! outcomes on the second serial port. +//! outcomes on the test terminal. //! //! Every helper and test that reads the machine's files or runs `uefisettings` calls //! [`require_test_vm`] first, so on any other machine the tests fail before they touch @@ -28,11 +28,11 @@ use std::fs; use std::fs::File; use std::io; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; +use std::ops::RangeInclusive; +use std::os::unix::fs::FileExt; use std::process::Command; use std::process::Stdio; +use std::slice; use serde::de::DeserializeOwned; use serde::Deserialize; @@ -52,6 +52,11 @@ pub const EFIVARS: &str = "/sys/firmware/efi/efivars"; pub const HIIDB_VARIABLE: &str = "/sys/firmware/efi/efivars/HiiDB-1b838190-4625-4ead-abc9-cd5e6af18fe0"; +/// The variable in which publish-hiidb puts the CRC-32 of the HII database that +/// [`HIIDB_VARIABLE`] points at. +pub const HIIDB_CRC32_VARIABLE: &str = + "/sys/firmware/efi/efivars/HiiDBCrc32-de145b29-c3cd-455f-91de-a76237b67c52"; + /// `Backend::Unknown` in `thrift/uefisettings_backend.thrift`. pub const BACKEND_UNKNOWN: i32 = 0; @@ -66,6 +71,10 @@ pub const DRIVER_SAMPLE_VARIABLE: &str = /// Size of the attributes that efivarfs puts before the data of a variable. pub const EFIVARFS_HEADER_SIZE: usize = 4; +/// Linux's EFAULT (`include/uapi/asm-generic/errno-base.h`), with which `/dev/mem`'s read() +/// refuses memory that the kernel does not map. +pub const EFAULT: i32 = 14; + /// The prefix of the answer of a question whose value the HII backend cannot read /// (`src/lib/hii/forms.rs`). pub const UNAVAILABLE: &str = " Option> { } } -/// Reads the HII database where the `HiiDB` variable points, through `/dev/mem`, as -/// `uefisettings` does. Panics if the variable does not exist. +/// The HII database that publish-hiidb published. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct PublishedDatabase { + /// Its length, from [`HIIDB_VARIABLE`]. + pub length: u32, + /// Its physical address, from [`HIIDB_VARIABLE`]. + pub address: u64, + /// Its CRC-32, from [`HIIDB_CRC32_VARIABLE`], for the tests to compare: read() of + /// `/dev/mem` serves the database only on x86 below the end of Linux's low memory, and + /// mmap() needs unsafe code, which this crate forbids. + pub crc32: u32, +} + +/// The HII database that [`HIIDB_VARIABLE`] and [`HIIDB_CRC32_VARIABLE`] describe. Panics +/// if either variable does not exist or has the wrong size. #[track_caller] -pub fn read_published_database() -> io::Result> { +pub fn published_database() -> PublishedDatabase { let hiidb = variable_data(HIIDB_VARIABLE).expect("the HiiDB variable does not exist"); let (length, address) = hiidb.split_at(size_of::()); - let length = u32::from_le_bytes(length.try_into().expect("HiiDB holds no length")); - let address = u64::from_le_bytes(address.try_into().expect("HiiDB holds no 64-bit address")); - let mut database = vec![0; length.try_into().expect("the length exceeds usize")]; - let mut mem = File::open("/dev/mem")?; - mem.seek(SeekFrom::Start(address))?; - mem.read_exact(&mut database)?; - Ok(database) + let crc32 = + variable_data(HIIDB_CRC32_VARIABLE).expect("the HiiDBCrc32 variable does not exist"); + PublishedDatabase { + length: u32::from_le_bytes(length.try_into().expect("HiiDB holds no length")), + address: u64::from_le_bytes(address.try_into().expect("HiiDB holds no 64-bit address")), + crc32: u32::from_le_bytes(crc32.try_into().expect("HiiDBCrc32 holds no CRC-32")), + } +} + +/// Reads the byte at physical `address` with read() of `/dev/mem`, which `uefisettings` tries +/// before mmap(). +#[track_caller] +pub fn read_dev_mem_byte(address: u64) -> io::Result { + require_test_vm(); + let mut byte = 0; + File::open("/dev/mem")?.read_exact_at(slice::from_mut(&mut byte), address)?; + Ok(byte) +} + +/// Whether `address` is in a System RAM range of `/proc/iomem`, which `/dev/mem` refuses to +/// read and map under `CONFIG_STRICT_DEVMEM` (`devmem_is_allowed()`), except in x86's first +/// MiB. +#[track_caller] +pub fn is_system_ram(address: u64) -> bool { + require_test_vm(); + let iomem = fs::read_to_string("/proc/iomem").expect("cannot read /proc/iomem"); + iomem + .lines() + .filter_map(|line| line.trim_start().strip_suffix(" : System RAM")) + .map(parse_iomem_range) + .any(|range| range.contains(&address)) +} + +/// Parses a range of `/proc/iomem`, `-` in hexadecimal. +fn parse_iomem_range(range: &str) -> RangeInclusive { + let bounds = range + .split_once('-') + .map(|(start, end)| (u64::from_str_radix(start, 16), u64::from_str_radix(end, 16))); + let Some((Ok(start), Ok(end))) = bounds else { + panic!("/proc/iomem: cannot parse the range {range:?}"); + }; + start..=end } /// The bytes that differ between `before` and `after`, which have the same length, as diff --git a/tests/qemu/guest/tests/boot_smoke.rs b/tests/qemu/guest/tests/boot_smoke.rs index 44be374..657f5f4 100644 --- a/tests/qemu/guest/tests/boot_smoke.rs +++ b/tests/qemu/guest/tests/boot_smoke.rs @@ -10,8 +10,8 @@ // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -//! A guest that boots without an HII fixture: OVMF publishes no HII database, so no backend -//! is detected and every HII command fails cleanly. +//! A guest that boots without an HII fixture: the firmware publishes no HII database, so no +//! backend is detected and every HII command fails cleanly. use std::fs; use std::io; diff --git a/tests/qemu/guest/tests/hii.rs b/tests/qemu/guest/tests/hii.rs index b83d8ef..5863501 100644 --- a/tests/qemu/guest/tests/hii.rs +++ b/tests/qemu/guest/tests/hii.rs @@ -27,7 +27,8 @@ use uefisettings_qemu_guest::driver_sample::ONE_OF_OFFSET; use uefisettings_qemu_guest::driver_sample::SERIAL_PORT; use uefisettings_qemu_guest::driver_sample::SERIAL_PORT_OFFSET; use uefisettings_qemu_guest::json; -use uefisettings_qemu_guest::read_published_database; +use uefisettings_qemu_guest::published_database; +use uefisettings_qemu_guest::read_dev_mem_byte; use uefisettings_qemu_guest::single_response; use uefisettings_qemu_guest::uefisettings; use uefisettings_qemu_guest::variable_data; @@ -36,8 +37,13 @@ use uefisettings_qemu_guest::Question; use uefisettings_qemu_guest::Responses; use uefisettings_qemu_guest::BACKEND_HII; use uefisettings_qemu_guest::DRIVER_SAMPLE_VARIABLE; +use uefisettings_qemu_guest::EFAULT; use uefisettings_qemu_guest::UNAVAILABLE; +/// Linux's SIGBUS on arm64 (`include/uapi/asm-generic/signal.h`). +#[cfg(target_arch = "aarch64")] +const SIGBUS: i32 = 7; + /// The questions of the tests with their default answers and options, and how many times /// the HII database lists each. const DEFAULTS: [(&str, &str, &[&str], usize); 4] = [ @@ -58,7 +64,7 @@ const DEFAULTS: [(&str, &str, &[&str], usize); 4] = [ /// Questions whose values the HII backend cannot read from Linux: on DriverSample's /// variable without runtime access, on its name/value store, and on the RAM disk form of -/// OVMF, which has no store. +/// the firmware, which has no store. const UNAVAILABLE_QUESTIONS: [&str; 4] = [ "How tall are you? (Hex)", "NameValueVar0", @@ -100,15 +106,61 @@ fn driver_sample_variable_holds_the_defaults() { #[test] fn extract_db_copies_the_published_database() { - let published = read_published_database() - .unwrap_or_else(|error| panic!("cannot read the published database: {error}")); + let published = published_database(); let path = "/tmp/hiidb.bin"; let output = uefisettings(&["hii", "extract-db", path]); assert_eq!(output.code, Some(0), "{output:#?}"); let extracted = fs::read(path).unwrap_or_else(|error| panic!("cannot read {path}: {error}")); - assert!( - extracted == published, - "{path} differs from the published database" + assert_eq!( + (extracted.len(), crc32fast::hash(&extracted)), + ( + usize::try_from(published.length).expect("the length exceeds usize"), + published.crc32 + ), + "{path} differs from the published database, as (length, CRC-32)" + ); +} + +/// read() of `/dev/mem` serves the database only on x86_64: arm64 Linux maps neither +/// runtime services data nor reserved memory, and 32-bit x86 Linux nothing above its low +/// memory, so there `extract_db_copies_the_published_database` passes only through the +/// fallback of `uefisettings` to mmap(). +#[test] +fn dev_mem_read_serves_the_database_only_on_x86_64() { + let address = published_database().address; + let errno = read_dev_mem_byte(address) + .err() + .map(|error| error.raw_os_error()); + let expected = if cfg!(target_arch = "x86_64") { + None + } else { + Some(Some(EFAULT)) + }; + assert_eq!(errno, expected, "read() of /dev/mem at {address:#x}: errno"); +} + +/// The emulator faults a 64-bit load from the mapped database, which publish-hiidb puts at an +/// unaligned address, as arm64 hardware faults unaligned wide loads from Device memory +/// (); so a load like that in `uefisettings` +/// fails `extract_db_copies_the_published_database`. +#[test] +#[cfg(target_arch = "aarch64")] +fn unaligned_wide_load_from_the_mapped_database_faults() { + use std::os::unix::process::ExitStatusExt; + use std::process::Command; + use std::process::Stdio; + + let address = published_database().address; + let status = Command::new("/bin/busybox") + .args(["devmem", &format!("{address:#x}"), "64"]) + .stdout(Stdio::null()) + .status() + .expect("cannot run busybox devmem"); + assert_eq!( + status.signal(), + Some(SIGBUS), + "devmem at {address:#x}: {status}; QEMU before 9.0 does not fault unaligned loads from \ + Device memory" ); } @@ -163,7 +215,7 @@ fn show_ifr_shows_the_formsets() { .filter_map(|line| line.trim_start().strip_prefix("OpCode: FormSet - ")) .filter_map(|line| line.split_once(" - GUID ").map(|(name, _)| name)) .collect(); - // DriverSample's two formsets and one of OVMF. + // DriverSample's two formsets and one of the firmware. for name in [ "Browser Testcase Engine", "ABC Information Sample", diff --git a/tests/qemu/guest/tests/hii_devmem_guard.rs b/tests/qemu/guest/tests/hii_devmem_guard.rs index e5b928d..5df602b 100644 --- a/tests/qemu/guest/tests/hii_devmem_guard.rs +++ b/tests/qemu/guest/tests/hii_devmem_guard.rs @@ -17,14 +17,15 @@ use std::fs; use std::io; +use uefisettings_qemu_guest::is_system_ram; use uefisettings_qemu_guest::json_error; -use uefisettings_qemu_guest::read_published_database; +use uefisettings_qemu_guest::published_database; use uefisettings_qemu_guest::require_test_vm; #[test] -fn dev_mem_refuses_to_read_the_database() { - let error = read_published_database().expect_err("/dev/mem served the database"); - assert_eq!(error.kind(), io::ErrorKind::PermissionDenied, "{error}"); +fn the_database_is_in_system_ram() { + let address = published_database().address; + assert!(is_system_ram(address), "{address:#x} is not System RAM"); } #[test] diff --git a/tests/qemu/init b/tests/qemu/init index ac175a4..5b494ef 100755 --- a/tests/qemu/init +++ b/tests/qemu/init @@ -14,8 +14,8 @@ # PID 1 of the QEMU test guests, which tests/qemu/run.sh puts in their initramfs. It # mounts what uefisettings needs, runs the test binary that uefisettings.test= on the # kernel command line names (with the libtest filter from uefisettings.filter=, if any) -# and then the health checks, both writing to the second serial port. It prints both exit -# codes on the console and powers off. +# and then the health checks, both writing to the terminal that uefisettings.tty= names. +# It prints both exit codes on the console and powers off. /bin/busybox --install -s /bin # uefisettings locks a file in /run/lock before it writes a setting. @@ -23,22 +23,28 @@ mkdir -p /dev /proc /run/lock /sys /tmp mount -t proc proc /proc mount -t sysfs sysfs /sys mount -t devtmpfs devtmpfs /dev +# The modules that run.sh copies, which the guest's kernel has not built in. +for module in /lib/modules/*.ko; do + [ -e "$module" ] && insmod "$module" +done mount -t efivarfs efivarfs /sys/firmware/efi/efivars test= filter= +tty= read -r cmdline < /proc/cmdline for parameter in $cmdline; do case $parameter in uefisettings.test=*) test=${parameter#*=} ;; uefisettings.filter=*) filter=${parameter#*=} ;; + uefisettings.tty=*) tty=/dev/${parameter#*=} ;; esac done -# The firmware writes to both serial ports and may leave a line unfinished, so the -# start and exit lines begin with a newline. run.sh prints the second port from the start -# line on. -printf '\nuefisettings-qemu-test: start %s %s\n' "$test" "$filter" > /dev/ttyS1 +# The firmware writes to the console and that terminal and may leave a line unfinished, so +# the start and exit lines begin with a newline. run.sh prints the terminal's output from +# the start line on. +printf '\nuefisettings-qemu-test: start %s %s\n' "$test" "$filter" > "$tty" # The health checks judge only the test: the kernel log and taint of the boot depend on # the host, e.g. a RETBleed error on Intel hosts that hide IBRS from the guest. read -r start < /proc/sys/kernel/random/uuid @@ -46,7 +52,7 @@ echo "uefisettings-qemu-test: start of test $start" > /dev/kmsg # cat, not read: busybox read reads a byte at a time, and numeric sysctls answer only a # read at offset 0, so read would get only the first digit. taint=$(cat /proc/sys/kernel/tainted /sys/kernel/warn_count) -"/tests/$test" --test-threads=1 ${filter:+"$filter"} > /dev/ttyS1 2>&1 +"/tests/$test" --test-threads=1 ${filter:+"$filter"} > "$tty" 2>&1 test_status=$? # The test added no taint (TAINT_* in include/linux/panic.h) or warning (a repeated WARN # adds no taint bit), and the kernel kept its log @@ -75,6 +81,6 @@ health_status=0 END { if (!found) print "cannot read the kernel log"; exit !found || error }' /tmp/kmsg || health_status=1 kill "$!" 2> /dev/null -} > /dev/ttyS1 2>&1 +} > "$tty" 2>&1 printf '\nuefisettings-qemu-test: exit %s %s\n' "$test_status" "$health_status" poweroff -f diff --git a/tests/qemu/kernel.sha256 b/tests/qemu/kernel.sha256 deleted file mode 100644 index a4418de..0000000 --- a/tests/qemu/kernel.sha256 +++ /dev/null @@ -1 +0,0 @@ -ab1418c266e2b4b26090ab13e96c249a6bbe7a6fadca09c4e98c9ff477a5d58f kernel-core-6.17.1-300.fc43.x86_64.rpm diff --git a/tests/qemu/publish-hiidb/build.rs b/tests/qemu/publish-hiidb/build.rs index fa88bcf..14b1ccd 100644 --- a/tests/qemu/publish-hiidb/build.rs +++ b/tests/qemu/publish-hiidb/build.rs @@ -10,7 +10,7 @@ // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -//! Links publish-hiidb as a boot service driver: OVMF loads the image of a +//! Links publish-hiidb as a boot service driver: EDK2's firmware loads the image of a //! `Driver####` load option only if it is a driver (`BmIsLoadOptionPeHeaderValid` //! in EDK2's `BmLoadOption.c`), and it runs those options before it starts the //! kernel that QEMU's `-kernel` passes it. diff --git a/tests/qemu/publish-hiidb/src/main.rs b/tests/qemu/publish-hiidb/src/main.rs index 7a62129..6d7d5c1 100644 --- a/tests/qemu/publish-hiidb/src/main.rs +++ b/tests/qemu/publish-hiidb/src/main.rs @@ -12,12 +12,16 @@ //! UEFI driver for the QEMU tests of the `uefisettings` HII backend, which a //! `Driver####` load option runs as `publish-hiidb.efi [rt-data|reserved|loader-data]`: -//! it copies the HII database into pages of that memory type and points the -//! volatile `HiiDB` variable at them as `{ u32 length; u64 address }`, the way +//! it copies the HII database into pages of that memory type, one byte in, and points +//! the volatile `HiiDB` variable at the copy as `{ u32 length; u64 address }`, the way //! OCP firmware does for `src/lib/hii/extract.rs`. The x86 EFI stub reports //! `rt-data` (the default) and `reserved` as E820 reserved ranges, which //! `/dev/mem` serves under `CONFIG_STRICT_DEVMEM`; `loader-data` becomes System //! RAM, which it refuses. +//! arm64 Linux maps neither `rt-data` nor `reserved`, so `/dev/mem` serves them only +//! to mmap(), as 32-bit x86 Linux does above its low memory. +//! The driver also puts the CRC-32 of the database in the volatile `HiiDBCrc32` +//! variable, for the tests to check what `uefisettings` extracts. #![cfg_attr(target_os = "uefi", no_std, no_main)] #![deny(unsafe_code)] @@ -51,6 +55,16 @@ const HIIDB_VENDOR: Guid = guid!("1b838190-4625-4ead-abc9-cd5e6af18fe0"); /// Attributes of that variable: volatile and readable at runtime. const HIIDB_ATTRIBUTES: VariableAttributes = VariableAttributes::BOOTSERVICE_ACCESS.union(VariableAttributes::RUNTIME_ACCESS); +/// Name of the variable that holds the CRC-32 of the database, for the tests +/// only, with the attributes of `HiiDB`. +const HIIDB_CRC32_NAME: &CStr16 = cstr16!("HiiDBCrc32"); +/// Vendor GUID of that variable, which the tests own. +const HIIDB_CRC32_VENDOR: Guid = guid!("de145b29-c3cd-455f-91de-a76237b67c52"); + +/// Where the copy starts in its pages: an unaligned start makes `uefisettings` +/// read the first page from an unaligned address, which arm64 Device memory +/// faults for wide loads (). +const COPY_OFFSET: usize = 1; /// A failed step and the firmware's status. type Failure = (&'static str, Status); @@ -59,7 +73,7 @@ type Failure = (&'static str, Status); /// build is a driver. #[cfg(not(target_os = "uefi"))] fn main() { - panic!("publish-hiidb is a UEFI driver: build it for x86_64-unknown-uefi"); + panic!("publish-hiidb is a UEFI driver: build it for a UEFI target"); } #[entry] @@ -93,8 +107,11 @@ fn memory_type() -> Option { } } -/// Copies the exported HII database into pages of `memory_type` and points -/// `HiiDB` at them. Returns the address and length of the copy. +/// Copies the exported HII database into pages of `memory_type` at `COPY_OFFSET` +/// and points `HiiDB` at the copy. Returns the address and length of the copy. +/// +/// It first puts the CRC-32 of the database in `HiiDBCrc32`, so that `HiiDB` +/// never exists without it. /// /// The pages are never freed: the OS reads them after boot. fn publish(memory_type: MemoryType) -> Result<(u64, u32), Failure> { @@ -104,22 +121,35 @@ fn publish(memory_type: MemoryType) -> Result<(u64, u32), Failure> { .map_err(|error| ("exporting the HII database", error.status()))?; let length = u32::try_from(export.len()) .map_err(|_| ("the HII database exceeds 4 GiB", Status::BAD_BUFFER_SIZE))?; - let pages = export.len().div_ceil(PAGE_SIZE); + + let crc32 = boot::calculate_crc32(&export) + .map_err(|error| ("computing the CRC-32 of the HII database", error.status()))?; + runtime::set_variable( + HIIDB_CRC32_NAME, + &VariableVendor(HIIDB_CRC32_VENDOR), + HIIDB_ATTRIBUTES, + &crc32.to_le_bytes(), + ) + .map_err(|error| ("setting the HiiDBCrc32 variable", error.status()))?; + + let pages = (COPY_OFFSET + export.len()).div_ceil(PAGE_SIZE); let base = boot::allocate_pages(AllocateType::AnyPages, memory_type, pages) .map_err(|error| ("allocating pages", error.status()))?; + let copy = base.as_ptr().wrapping_add(COPY_OFFSET); #[expect( unsafe_code, reason = "the pages from allocate_pages() are only reachable through a raw pointer" )] // SAFETY: `allocate_pages` returned `pages * PAGE_SIZE` bytes that nothing // else references, and UEFI identity-maps memory, so the address is a - // valid pointer. `export` fits in them and does not overlap them. + // valid pointer. `export` fits in them after `COPY_OFFSET` and does not + // overlap them. unsafe { ptr::write_bytes(base.as_ptr(), 0, pages * PAGE_SIZE); - ptr::copy_nonoverlapping(export.as_ptr(), base.as_ptr(), export.len()); + ptr::copy_nonoverlapping(export.as_ptr(), copy, export.len()); } - // `usize` is 64 bits wide on x86_64-unknown-uefi. - let address = base.as_ptr().addr() as u64; + // `usize` has at most 64 bits on the UEFI targets. + let address = copy.addr() as u64; let mut data = [0; size_of::() + size_of::()]; data[..4].copy_from_slice(&length.to_le_bytes()); diff --git a/tests/qemu/run.sh b/tests/qemu/run.sh index c5223e7..91bd792 100755 --- a/tests/qemu/run.sh +++ b/tests/qemu/run.sh @@ -12,33 +12,85 @@ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. # Runs the QEMU tests: boots each scenario of tests/qemu/scenarios.tsv, or only the named -# ones, under OVMF and fails unless the guest tests and health checks of every boot pass. -# tests/qemu/README.md describes the requirements and the logs. +# ones, under EDK2's firmware and fails unless the guest tests and health checks of every +# boot pass. tests/qemu/README.md describes the requirements and the logs. # -# Usage: tests/qemu/run.sh [SCENARIO...] +# Usage: tests/qemu/run.sh ARCH [SCENARIO...] # -# Environment variables, which default to the paths of the Ubuntu 24.04 packages: -# QEMU the qemu-system-x86_64 to run (qemu-system-x86_64 from PATH), or a wrapper -# that execs it: the timeout of a boot signals that program, not its children -# OVMF_CODE OVMF code image (/usr/share/OVMF/OVMF_CODE_4M.fd) -# OVMF_VARS OVMF variable store, which each scenario copies (/usr/share/OVMF/OVMF_VARS_4M.fd) -# BUSYBOX static busybox for the initramfs (/usr/bin/busybox) -# DRIVER_SAMPLE_DIR the directory of DriverSample.efi, which build-driver-sample.sh -# fills if the file is missing (target/qemu-test/driver-sample) +# ARCH is x86_64, i686 or aarch64. The guest's kernel, firmware and busybox are the +# packages of tests/qemu/guest-packages-ARCH. QEMU is the qemu-system-* of the table below +# from PATH, or a wrapper that execs it: the timeout of a boot signals that program, not +# its children. set -euo pipefail -qemu=${QEMU:-qemu-system-x86_64} -ovmf_code=${OVMF_CODE:-/usr/share/OVMF/OVMF_CODE_4M.fd} -ovmf_vars=${OVMF_VARS:-/usr/share/OVMF/OVMF_VARS_4M.fd} -busybox=${BUSYBOX:-/usr/bin/busybox} +if (($# < 1)); then + echo "usage: $0 ARCH [SCENARIO...]" >&2 + exit 2 +fi +arch=$1 +shift +# The test terminal is the second serial port, but QEMU's virt machine gives the guest one +# PL011 UART, so the aarch64 guests write to a virtio console. Debian's i686 kernel has +# efivarfs as a module. aarch64 runs under TCG, so it boots on x86_64 hosts like the others. +case $arch in +x86_64) + qemu=(qemu-system-x86_64 -machine q35 -accel kvm -cpu host) + firmware=usr/share/OVMF/OVMF_CODE_4M.fd + firmware_vars=usr/share/OVMF/OVMF_VARS_4M.fd + vmlinuz_pattern='lib/modules/*/vmlinuz' + modules=() + console=ttyS0 + tty=ttyS1 + tty_device=(-serial chardev:tty) + edk2_arch=X64 + kernel_params=() + ;; +i686) + qemu=(qemu-system-i386 -machine q35 -accel kvm -cpu host) + firmware=usr/share/OVMF/OVMF32_CODE_4M.fd + firmware_vars=usr/share/OVMF/OVMF32_VARS_4M.fd + vmlinuz_pattern='boot/vmlinuz-*' + modules=('lib/modules/*/kernel/fs/efivarfs/efivarfs.ko') + console=ttyS0 + tty=ttyS1 + tty_device=(-serial chardev:tty) + edk2_arch=IA32 + # Containment: GitHub's AMD runners' KVM (linux-azure) clears CR3 bits 0-11 of a vCPU + # without CR4.PCIDE at each NPT root load (https://bugs.launchpad.net/bugs/2106673); + # in PAE paging bits 5-11 address the PDPT, which Debian 12's kernel takes from a + # 32-byte slab unless pti=on (Linux v6.1 _pgd_alloc; SHARED_KERNEL_PMD is !PTI). + kernel_params=(pti=on) + ;; +aarch64) + qemu=(qemu-system-aarch64 -machine virt -accel tcg -cpu cortex-a57) + firmware=usr/share/AAVMF/AAVMF_CODE.fd + firmware_vars=usr/share/AAVMF/AAVMF_VARS.fd + vmlinuz_pattern='lib/modules/*/vmlinuz' + modules=() + console=ttyAMA0 + tty=hvc0 + tty_device=(-device virtio-serial-pci -device "virtconsole,chardev=tty") + edk2_arch=AARCH64 + kernel_params=() + # cc, the default, links for the host. Set the variable for another linker: it + # overrides cargo's config files, as this default does. + export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=${CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER-aarch64-linux-gnu-gcc} + ;; +*) + echo "unknown architecture $arch; the architectures are x86_64, i686 and aarch64" >&2 + exit 2 + ;; +esac here=$(realpath "$(dirname "$0")") repo=$(realpath "$here/../..") out=$repo/target/qemu-test -driver_sample=${DRIVER_SAMPLE_DIR:-$out/driver-sample} -run=$out/$(date -u +%Y%m%dT%H%M%SZ) -target=x86_64-unknown-linux-musl +driver_sample=$out/$arch/driver-sample +run=$out/$arch/$(date -u +%Y%m%dT%H%M%SZ) +# Unpacked again by every run, which keeps one copy per architecture. +guest=$out/$arch/guest +target=$arch-unknown-linux-musl # Every row has the 4 columns that the header of scenarios.tsv lists and the loop below # reads, none of them empty, so that no row silently boots nothing. @@ -61,10 +113,9 @@ mapfile -t named < <(awk -F '\t' '$1 !~ /^(#|$)/ { }' "$here/scenarios.tsv") echo "run.sh: start $(date -u +%FT%TZ), logs in $run" -mkdir -p "$run/initramfs/bin" "$run/initramfs/tests" "$run/esp" +mkdir -p "$run/initramfs/bin" "$run/initramfs/lib/modules" "$run/initramfs/tests" "$run/esp" grep --max-count=1 '^model name' /proc/cpuinfo -"$qemu" --version -"$busybox" --help | sed -n 1p +"${qemu[0]}" --version # virt-fw-vars has no --version option, so this asks the Python that its first line names, # the one that runs it, for the version of virt-firmware. Another first line, or a failure, @@ -91,7 +142,7 @@ executables() { cargo build --release --locked --target "$target" --bin uefisettings \ --manifest-path "$repo/Cargo.toml" --message-format=json-render-diagnostics | executables >"$run/executables" -cargo build --release --locked --target x86_64-unknown-uefi \ +cargo build --release --locked --target "$arch-unknown-uefi" \ --manifest-path "$here/publish-hiidb/Cargo.toml" --message-format=json-render-diagnostics | executables >>"$run/executables" cargo test --release --locked --target "$target" --features test-vm --test '*' --no-run \ @@ -100,7 +151,8 @@ cargo test --release --locked --target "$target" --features test-vm --test '*' - # The initramfs gets uefisettings, the test binaries, busybox and init. The ESP gets the # HII fixture: publish-hiidb and EDK2's DriverSample. unnamed=() -while read -r name path; do +# read fails on a last line without a newline, which still counts. +while read -r name path || [[ -n $name ]]; do case $name in uefisettings) cp "$path" "$run/initramfs/bin/" ;; publish-hiidb) cp "$path" "$run/esp/" ;; @@ -114,22 +166,34 @@ if ((${#unnamed[@]} > 0)); then echo "no scenario boots the test binaries ${unnamed[*]}; add them to scenarios.tsv" >&2 exit 1 fi -cp "$busybox" "$run/initramfs/bin/busybox" cp "$here/init" "$run/initramfs/init" if [[ ! -f $driver_sample/DriverSample.efi ]]; then - "$here/build-driver-sample.sh" "$driver_sample" + "$here/build-driver-sample.sh" "$edk2_arch" "$driver_sample" fi cp "$driver_sample/DriverSample.efi" "$run/esp/" -"$here/fetch-kernel.sh" "$out/kernel-rpms" "$run/kernel" -vmlinuz=$(echo "$run"/kernel/lib/modules/*/vmlinuz) -if [[ ! -f $vmlinuz ]]; then - echo "expected the vmlinuz of exactly one kernel in $run/kernel" >&2 - exit 1 -fi +rm -rf "$guest" +"$here/fetch-packages.sh" "$here/guest-packages-$arch" "$out/packages" "$guest" +# Prints the one file of the guest's packages that a pattern of the table matches. +guest_file() { + local files + # Only the pattern globs, not the path of the repository. + mapfile -t files < <(cd "$guest" && compgen -G "$1") + if ((${#files[@]} != 1)) || [[ ! -f $guest/${files[0]} ]]; then + echo "expected exactly one $1 in $guest" >&2 + return 1 + fi + echo "$guest/${files[0]}" +} +vmlinuz=$(guest_file "$vmlinuz_pattern") +for module in "${modules[@]}"; do + cp "$(guest_file "$module")" "$run/initramfs/lib/modules/" +done +cp "$guest/usr/bin/busybox" "$run/initramfs/bin/busybox" failed=() -while IFS=$'\t' read -r -u 3 name drivers variables boots; do +# read fails on a last line without a newline, which still counts. +while IFS=$'\t' read -r -u 3 name drivers variables boots || [[ -n $name ]]; do [[ -z $name || $name == \#* ]] && continue (($# == 0)) || [[ " $* " == *" $name "* ]] || continue dir=$run/$name @@ -140,7 +204,7 @@ while IFS=$'\t' read -r -u 3 name drivers variables boots; do seed=() if [[ $drivers != - ]]; then IFS=';' read -r -a files <<<"$drivers" - virt-fw-vars --input "$ovmf_vars" "${files[@]/#/--append-boot-filepath=}" \ + virt-fw-vars --input "$guest/$firmware_vars" "${files[@]/#/--append-boot-filepath=}" \ --output-json "$dir/boot.json" 2>"$dir/virt-fw-vars.log" jq '.variables[].name |= sub("^Boot"; "Driver")' "$dir/boot.json" >"$dir/drivers.json" seed+=("$dir/drivers.json") @@ -148,8 +212,8 @@ while IFS=$'\t' read -r -u 3 name drivers variables boots; do [[ $variables == - ]] || seed+=("$here/vars/$variables") jq --null-input '{version: 2, variables: [inputs.variables[]]}' "${seed[@]}" \ "$dir/vars.json" - virt-fw-vars --input "$ovmf_vars" --set-json "$dir/vars.json" --output "$dir/vars.fd" \ - 2>>"$dir/virt-fw-vars.log" + virt-fw-vars --input "$guest/$firmware_vars" --set-json "$dir/vars.json" \ + --output "$dir/vars.fd" 2>>"$dir/virt-fw-vars.log" boot=0 IFS=, read -r -a specs <<<"$boots" for spec in "${specs[@]}"; do @@ -161,27 +225,42 @@ while IFS=$'\t' read -r -u 3 name drivers variables boots; do echo "run.sh: $name boot $boot: $spec" # Without --foreground, timeout would move QEMU into a process group of its own, # which a timeout around make does not reach. - timeout --foreground -k 10 120 "$qemu" -nodefaults -no-user-config -no-reboot \ - -display none -sandbox on -machine q35 -accel kvm -cpu host -m 1024 -smp 2 \ - -drive "if=pflash,format=raw,unit=0,readonly=on,file=$ovmf_code" \ + # The QEMU log, empty if QEMU does not start, tells how the guest ended (QEMU 10.2): + # a poweroff makes a shutdown request (reason=6); a reset, after which -no-reboot + # just ends QEMU, and a kill by the timeout make none. A CPU's last KVM exit is 8 + # (KVM_EXIT_SHUTDOWN) after its triple fault and 2 (I/O) after a reset through a port. + : >"$log.qemu.log" + timeout --foreground -k 10 120 "${qemu[@]}" -nodefaults -no-user-config -no-reboot \ + -display none -sandbox on -m 1024 -smp 2 \ + -drive "if=pflash,format=raw,unit=0,readonly=on,file=$guest/$firmware" \ -drive "if=pflash,format=raw,unit=1,file=$dir/vars.fd" \ -drive "if=virtio,format=raw,readonly=on,file=fat:$run/esp" \ -kernel "$vmlinuz" -initrd "$dir/initrd.img" \ - -append "console=ttyS0 quiet panic=-1 uefisettings.test=$test${filter:+ uefisettings.filter=$filter}" \ - -serial "file:$log.console.log" -serial "file:$log.test.log" \ - max) max = cpu } + END { for (cpu = 0; cpu <= max; cpu++) if (cpu in last) print last[cpu] }' \ + "$log.qemu.log" >"$log.qemu.log.new" + mv "$log.qemu.log.new" "$log.qemu.log" sed -n '/^uefisettings-qemu-test: start/,$p' "$log.test.log" # init prints the exit codes of the tests and the health checks last. A filter # that matches no test passes too, so the tests must also report a passed test. if ! tr -d '\r' <"$log.console.log" | grep --quiet --line-regexp 'uefisettings-qemu-test: exit 0 0' || ! grep --quiet '^test result: ok\. [1-9]' "$log.test.log"; then failed+=("$name boot $boot") + sed "s/^/run.sh: $name boot $boot: QEMU: /" "$log.qemu.log" fi done done 3<"$here/scenarios.tsv" echo "run.sh: end $(date -u +%FT%TZ), logs in $run" if ((${#failed[@]} > 0)); then - printf 'run.sh: FAILED: %s; see its console.log and test.log\n' "${failed[@]}" + printf 'run.sh: FAILED: %s; see its console.log, test.log and qemu.log\n' "${failed[@]}" exit 1 fi diff --git a/tests/qemu/ubuntu-packages b/tests/qemu/ubuntu-packages index cc835e5..ace8a5c 100644 --- a/tests/qemu/ubuntu-packages +++ b/tests/qemu/ubuntu-packages @@ -1,13 +1,16 @@ build-essential -busybox-static cpio curl +gcc-aarch64-linux-gnu git jq +libc6-dev-arm64-cross +libc6-dev-i386 nasm -ovmf python3 python3-virt-firmware +qemu-system-arm qemu-system-x86 rpm2cpio uuid-dev +zstd