diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..be2531cf --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,350 @@ +# Stackinator + +Stackinator is a Python CLI tool that generates build configurations for scientific software stacks on HPE Cray EX (Alps) systems. It acts like `cmake`/`configure`: given a **recipe** and a **cluster configuration**, it produces a build directory with Makefiles and Spack YAML files. The actual build is then performed by `make`. + +## Two-Phase Workflow + +``` +stack-config -b BUILD -r RECIPE -s SYSTEM [--mirror MIRRORS] [-m MOUNT] # [-c CACHE] is legacy + → generates BUILD/ directory with Makefiles + spack.yaml files + +cd BUILD +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make store.squashfs -j64 + → clones Spack, concretises, builds, creates store.squashfs +``` + +The `store.squashfs` SquashFS image is the final artifact, intended to be mounted at the recipe's `store` path (default `/user-environment`) as a uenv image. + +## Repository Layout + +``` +stackinator/ # Python package + main.py # CLI entry point (stack-config) + recipe.py # Recipe class: parses and validates all recipe YAML + builder.py # Builder class: writes all files to the build path + mirror.py # Mirrors class: validates the --mirror mirrors.yaml, emits spack mirror/cache config + spack_util.py # Tiny helper: checks if a path is a spack package repo + schema.py # JSON schema validators with default-injection + schema/ # JSON schemas for each YAML file type + config.json + compilers.json + environments.json + mirror.json # mirrors.yaml schema (buildcache/bootstrap/sourcemirror/sourcecache/concretizer) + cache.json # legacy -c/--cache cache.yaml schema + modules.json + templates/ # Jinja2 templates for all generated files + Makefile # Top-level build orchestration + Makefile.compilers # Compiler build steps + Makefile.environments # Environment build + view generation steps + Makefile.generate-config # Generates upstream spack config for the uenv + Make.user # Build path / store / sandbox variable definitions + repos.yaml # Generated spack repos.yaml + stack-debug.sh # Debug helper script + compilers.*.spack.yaml # Per-compiler spack.yaml configs + environments.spack.yaml # Environment spack.yaml config + etc/ + Make.inc # Shared make rules (concretize, depfile, compiler_bin_dirs) + bwrap-mutable-root.sh # Bubblewrap sandbox wrapper + envvars.py # CLI tool: generates env.json for views and uenv metadata +docs/ # MkDocs documentation source +unittests/ # pytest test suite + test_schema.py # Schema validation tests (primary test coverage) + recipes/ # Example recipes used by tests + yaml/ # Example YAML snippets for testing +``` + +## Recipe Format (input) + +A recipe is a directory containing YAML files: + +### `config.yaml` (required) +```yaml +name: prgenv-gnu +store: /user-environment # mount point; default /user-environment +version: 2 # must be 2 for Spack 1.0 (Stackinator 6+) +spack: + repo: https://github.com/spack/spack.git + commit: releases/v1.0 # branch, tag, or SHA; null = default branch + packages: + repo: https://github.com/spack/spack-packages.git + commit: develop +description: "optional text" +default-view: develop # optional: view loaded when no view is specified +``` + +- `version: 1` (the default) targets Spack v0.23 and is only supported by Stackinator v5 (`releases/v5` branch). **Current `main` requires `version: 2`.** +- `store` can be overridden at configure time with `-m/--mount`. + +### `compilers.yaml` (required) +```yaml +gcc: + version: "13" # required; must be quoted string +nvhpc: # optional + version: "25.1" +llvm: # optional + version: "16" +llvm-amdgpu: # optional + version: "6.0" +intel-oneapi-compilers: # optional + version: "2024.1" +``` + +Build order: `gcc` is built first (using system compiler), then `nvhpc`/`llvm`/`llvm-amdgpu`/`intel-oneapi-compilers` are built using the gcc toolchain. Stackinator appends opinionated variants (e.g. `gcc@13 +bootstrap`, `nvhpc@25.1 ~mpi~blas~lapack`, `llvm@16 +clang ~gold`). + +### `environments.yaml` (required) +```yaml +my-env: + compiler: [gcc] # required; list from compilers.yaml keys; first = default + specs: # required; list of spack specs + - cmake + - hdf5+mpi + network: # optional; null = no MPI + mpi: cray-mpich # full spack spec for MPI (cray-mpich or openmpi) + specs: ['libfabric@1.22'] # optional; overrides network.yaml defaults + unify: true # concretizer: true | false | when_possible (default true) + duplicates: + strategy: minimal # minimal | full | none (default minimal) + deprecated: false # allow deprecated spack versions (default false) + variants: # applied to all packages (packages:all:variants) + - +cuda + - cuda_arch=80 + prefer: null # packages:all:prefer; auto-set if null + packages: # external packages to discover via `spack external find` + - perl + - git + views: # optional filesystem views + default: null # view name → view config (null = defaults) + no-python: + exclude: [python] + uenv: + add_compilers: true # default true; adds compiler symlinks to view/bin + prefix_paths: + LD_LIBRARY_PATH: [lib, lib64] + env_vars: + set: + - MYVAR: "value" + - MYVAR2: null # unsets the variable + prepend_path: + - PATH: "/some/path" + append_path: + - PKG_CONFIG_PATH: "/usr/lib/pkgconfig" +``` + +**Key constraints:** +- Do not include MPI or compilers in `specs`; they are handled by `network.mpi` and `compiler`. +- Spec matrices are not supported. +- Only one MPI per environment; create separate environments for multiple MPIs. +- The `prefer` field is auto-generated if `null`: it nudges Spack to use the first compiler for all packages. + +#### Environment variable special syntax +- `${@VAR@}` — deferred expansion: expands `VAR` at uenv load time (e.g. `${@HOME@}`) +- `$@key@` — substitution at configure time: `mount`, `view_name`, `view_path` + +#### Supported prefix-path variables (hardcoded in `etc/envvars.py`) +`ACLOCAL_PATH`, `CMAKE_PREFIX_PATH`, `CPATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, `MANPATH`, `MODULEPATH`, `PATH`, `PKG_CONFIG_PATH`, `PYTHONPATH` + +### `modules.yaml` (optional) +Presence of this file enables module generation. Follows Spack's module config format with two differences: +- `modules:default:arch_folder` must be `false` (Stackinator doesn't support `true`) +- `modules:default:roots:tcl` is ignored and overwritten by Stackinator + +### `packages.yaml` (optional) +Standard Spack `packages.yaml` with recipe-specific external package overrides. + +### `repo/` (optional) +Custom Spack package definitions. Must contain a `packages/` subdirectory. +Merged into a single `alps` namespace repo alongside system and site packages. +Precedence: recipe repo > site repos (from cluster config `repos.yaml`) > Spack builtin. + +### `post-install` / `pre-install` (optional) +Shell scripts (any language) run inside the bwrap sandbox: +- `pre-install`: after Spack is set up, before first compiler build +- `post-install`: after all packages are built, before squashfs generation + +Both are Jinja-templated with variables: `env.mount`, `env.config`, `env.build`, `env.spack`. + +### `extra/` (optional) +Arbitrary files copied to `meta/extra/` in the final image (used for CI metadata). + +## Cluster Configuration (input) + +A directory (passed via `-s/--system`) containing: + +``` +cluster-config/ + packages.yaml # Spack external packages; must include gcc + network.yaml # MPI defaults and network library package configs + repos.yaml # optional; list of relative paths to site-wide spack repos +``` + +Mirror/cache config is **not** part of the cluster configuration — it is supplied separately with `--mirror` (a `mirrors.yaml` here is rejected). See [Mirrors and Build Caches](#mirrors-and-build-caches). + +`network.yaml` structure: +```yaml +mpi: + cray-mpich: + specs: [libfabric@1.22] # default specs injected when cray-mpich is chosen + openmpi: + specs: [libfabric@2.2.0] +packages: # standard spack packages.yaml content + libfabric: ... + cray-mpich: ... +``` + +Package precedence (recipe.py merges these): recipe `packages.yaml` > `network.yaml` packages > `packages.yaml` (minus gcc). The `gcc` entry from `packages.yaml` is isolated and used only for the gcc compiler build step. + +## Build Directory Structure (output) + +``` +BUILD/ + Makefile # top-level orchestration + Make.user # variables: BUILD_ROOT, STORE, SANDBOX, etc. + Make.inc # shared make rules (copied from etc/) + bwrap-mutable-root.sh # sandbox wrapper (copied from etc/) + envvars.py # view/meta generator (copied from etc/) + spack/ # cloned Spack repository + spack-packages/ # cloned spack-packages repository + config/ # global spack configuration scope + packages.yaml + repos.yaml + mirrors.yaml # if --mirror provided: buildcache/sourcemirror entries + config.yaml # if --mirror provided with a sourcecache (config:source_cache) + concretizer.yaml # if --mirror provided with a concretizer cache (concretizer:concretization_cache), spack >= 1.1 + bootstrap.yaml # if --mirror provided with a bootstrap mirror + key_store/ # if --mirror provided with gpg keys (decoded *.gpg) + compilers/ + Makefile + gcc/ + spack.yaml + packages.yaml # generated by spack external find + nvhpc/ # if nvhpc in recipe + spack.yaml + environments/ + Makefile + my-env/ + spack.yaml + generate-config/ # generates the upstream spack config for the final image + Makefile + modules/ # only if modules.yaml in recipe + modules.yaml + store/ # installation root (bind-mounted to recipe.store during build) + meta/ + configure.json # build metadata + env.json.in # view metadata template + recipe/ # copy of the recipe + repos/spack_repo/alps/ # consolidated custom package repo + repos/spack_repo/builtin/ # copy of spack builtin repo + env/ # filesystem views (created during build) + view-name/ + activate.sh + env.json + bin/ lib/ ... + store.squashfs # final compressed image + stack-debug.sh # debug helper: opens shell in build environment +``` + +## Python Architecture + +### `Recipe` class (`recipe.py`) +Parses and validates all recipe inputs in `__init__`. Key responsibilities: +- Validates each YAML file against its JSON schema (with default injection) +- Merges packages from cluster config, network.yaml, and recipe +- Generates full compiler specs (e.g. `gcc@13 +bootstrap`) from `compilers.yaml` +- Processes environments: resolves MPI specs from `network.yaml` templates, sets default `prefer` constraints, builds view metadata +- Provides `compiler_files` and `environment_files` properties (Jinja-rendered Makefiles and spack.yaml files) + +### `Builder` class (`builder.py`) +Writes all files to the build path. Key responsibilities: +- Creates directory structure +- Clones Spack and spack-packages repositories +- Merges and writes the consolidated `alps` spack package repo +- Renders all Jinja templates into build path files +- Writes metadata JSON files + +### `Mirrors` class (`mirror.py`) +A clean exemplar of the "recipe validates & renders, builder just prints" pattern. Constructed by `Recipe` from the `--mirror` file path; does ALL mirror input processing eagerly in `__init__` (loads + schema-validates `mirrors.yaml`, validates mirror urls, decodes/validates gpg keys to in-memory bytes, checks cache paths are absolute and expands env vars). Then presents pure static artifacts: +- typed members: `buildcache`, `bootstrap`, `source_mirrors`, `source_cache`, `concretizer_cache` +- `config_files(config_root) -> {abs_path: bytes}` — the `mirrors.yaml`, `config.yaml`, `concretizer.yaml`, `bootstrap.yaml`, and gpg key files the builder writes verbatim +- `gpg_key_paths(config_root)` and the `build_cache_mirror` / `push_to_build_cache` properties (the latter is `None` for a keyless, read-only build cache) + +Mirror/cache config is supplied ONLY via `--mirror`; a `mirrors.yaml` found in the system config dir is rejected with an error (it was never a system-config artifact). Relative gpg-key paths resolve against the `--mirror` file's own directory. + +### `schema.py` +JSON schema validation using `jsonschema`. The `validator()` function extends the validator to auto-inject `default` values from schemas into parsed instances, so downstream code can rely on optional fields always being present. + +### `etc/envvars.py` +A standalone CLI tool (copied into the build directory) with two subcommands: +- `envvars.py view [--compilers] [--prefix_paths]`: reads a Spack-generated `activate.sh`, parses env vars, adds compiler symlinks and prefix paths, writes `env.json` for the view +- `envvars.py uenv [--modules] [--spack]`: merges view `env.json` files with recipe `env_vars` config, writes the final `meta/env.json` + +The `EnvVarSet` class in `envvars.py` is also imported by `recipe.py` for processing `env_vars` at configure time. + +## Build Pipeline (Make targets) + +The top-level `Makefile` orchestrates in order: +1. `spack-setup` — sanity check, bootstrap concretizer +2. `pre-install` — run `pre-install-hook` if provided +3. `mirror-setup` — trust build-cache/mirror gpg keys (`cache-force` force-pushes built packages) +4. `compilers` — build gcc, then nvhpc/llvm/etc. (parallel within each stage) +5. `environments` — build all user environments (parallel) +6. `generate-config` — generate the upstream spack config files for the installed image +7. `modules-done` — generate TCL module files (if `modules.yaml` present) +8. `env-meta` — run `envvars.py uenv` to produce final `meta/env.json` +9. `post-install` — run `post-install-hook` if provided +10. `store.squashfs` — create the final squashfs image + +Key Make.inc rules: +- `%/spack.lock`: concretize a spack environment +- `%/Makefile`: generate a depfile from a lock file (enables parallel package builds) +- `compiler_bin_dirs`: helper to find compiler binaries given install prefixes + +The build runs inside a bwrap sandbox (`bwrap-mutable-root.sh`) that: +- Bind-mounts `BUILD/store` → `STORE` (the recipe mount point) +- Bind-mounts `BUILD/tmp` → `/tmp` +- Puts a tmpfs over `$HOME` (isolates user config) + +## Mirrors and Build Caches + +Spack mirrors and caches are configured in a single `mirrors.yaml` supplied with `stack-config --mirror ` (see `docs/build-caches.md` for the full reference). It can describe five optional entities: + +- **`buildcache`** (one): binary cache of built packages — the big build-time speed up. With a `private_key` it signs and pushes packages it builds; without one it is read-only (fetch only). `mount_specific: true` stores binaries in a per-mount-point subdir (Spack binaries embed the install prefix, so each mount point needs its own cache). Packages are pushed per-environment after a successful build; `cuda`, `nvhpc`, `perl` are excluded from pushes. `make cache-force` force-pushes everything built so far. +- **`bootstrap`** (one): for bootstrapping Spack itself (clingo etc.). The `url` is either a local `spack bootstrap mirror` directory (referenced via its own `metadata/sources`+`metadata/binaries`) or a remote url (source-only). Needs **no key** (bootstrap binaries are sha256-verified) → emitted as `config/bootstrap.yaml` (+ a generated `metadata.yaml` only for the remote case); it is NOT a `mirrors.yaml` entry. +- **`sourcemirror`** (many): read-only mirrors providing package source archives. +- **`sourcecache`** (one): a writable local dir Spack fills as it fetches sources → emitted as `config:source_cache`. +- **`concretizer`** (one): a writable local dir persisting Spack's **concretization results** → emitted as `concretizer:concretization_cache:{enable,url}` in `config/concretizer.yaml`. Useful to persist across ephemeral builds. The config key requires Spack ≥ 1.1; Stackinator infers the Spack version from `config.yaml:spack.commit` (via `Recipe.find_spack_version`, defaulting to the latest supported version when the commit can't be pinned) and **skips the cache with a warning** for Spack 1.0 (which rejects the key). + +`sourcecache` is emitted to `config/config.yaml`; `concretizer` to `config/concretizer.yaml`; `bootstrap` to `config/bootstrap.yaml`; `buildcache`/`sourcemirror` to `config/mirrors.yaml` (+ decoded keys under `config/key_store/`). + +**Legacy:** a binary cache can still be configured with a `cache.yaml` (`root` + optional `key`) passed to `-c/--cache`. This path is deprecated in favour of a `buildcache` entry and will be removed. + +## Testing + +```bash +uv run pytest # run tests +./lint # ruff format + ruff check --fix +``` + +Tests live in `unittests/test_schema.py` and cover schema validation and default injection. Test recipes are in `unittests/recipes/`, example YAML in `unittests/yaml/`. + +The test coverage is limited — the schema validators and their default-injection are well tested, but `Recipe`, `Builder`, and `envvars.py` have minimal test coverage. + +## Code Style + +- Python 3.12+ +- Linting: `ruff` (line length 120, E + F rules, E203 ignored) +- Format: `ruff format` +- Run both via `./lint` + +## Key Invariants and Pitfalls + +- **Build path restrictions**: cannot be in `/tmp`, `$HOME`, or root `/`. The bwrap sandbox rebinds these. +- **Version 2 is required**: `config.yaml` must have `version: 2` for current `main`. Version 1 recipes require the `releases/v5` branch. +- **gcc is required**: `packages.yaml` in cluster config must define an external `gcc`. It is handled separately from other system packages for the bootstrap build step. +- **MPI validation**: the MPI name in `network.mpi` must match a key in `network.yaml:mpi` templates from the cluster config. Unknown MPI implementations raise an error. +- **View names are globally unique**: view names must be unique across all environments in a recipe. +- **Mirror/cache config comes only from `--mirror`**: never from the recipe or the system/cluster config. A `mirrors.yaml` in the system config dir is rejected with an error. +- **Read-only build cache**: a `buildcache` without a `private_key` is fetched from but never pushed to (`push_to_build_cache` is `None`). +- **Schema-injected defaults**: every non-required field has a `default` in its JSON schema, injected at every level (including `additionalProperties` maps). Rely on fields always being present and check `is None` — do not use `.get()` to guard existence. +- **`default-view` must exist**: if set in `config.yaml`, the named view must be defined in `environments.yaml` (or be `modules`/`spack`). +- **`prefer` is auto-set**: if `null` in the recipe, Stackinator generates a `prefer` constraint using Spack's `%[when=...]` syntax to pin the default compiler. +- **Spack `uenv_tools` environment**: an internal environment named `uenv_tools` is injected into every build to install `squashfs`. Recipe authors must not use this name. diff --git a/README.md b/README.md index 5c7e0408..265b0ac3 100644 --- a/README.md +++ b/README.md @@ -5,3 +5,17 @@ A tool for building a scientific software stack from a recipe for vClusters on C Read the [documentation](https://eth-cscs.github.io/stackinator/) to get started. Create a ticket in our [GitHub issues](https://github.com/eth-cscs/stackinator/issues) if you find a bug, have a feature request or have a question. + +## running tests: + +Use uv to run the tests, which will in turn ensure that the correct dependencies from `pyproject.toml` are used: + +``` +uv run pytest +``` + +Before pushing, apply the linting rules (this calls uv under the hood): + +``` +./lint +``` diff --git a/bin/stack-config b/bin/stack-config index 66496991..342a82df 100755 --- a/bin/stack-config +++ b/bin/stack-config @@ -1,7 +1,8 @@ -#!/usr/bin/env -S uv run --script +#!/usr/bin/env -S uv run --no-refresh --script # /// script # requires-python = ">=3.12" # dependencies = [ +# "python-magic", # "jinja2", # "jsonschema", # "pyYAML", diff --git a/docs/build-caches.md b/docs/build-caches.md deleted file mode 100644 index 709f12be..00000000 --- a/docs/build-caches.md +++ /dev/null @@ -1,101 +0,0 @@ -# Build Caches - -Stackinator facilitates using Spack's binary build caches to speed up image builds. -Build caches are essential if you plan to build images regularly, as they generally lead to a roughly 10x speed up. -This is the difference between half an hour or 3 minutes to build a typical image. - -## Using Build caches - -To use a build cache, create a simple YAML file: - -```yaml title='cache-config.yaml' -root: $SCRATCH/uenv-cache -key: $SCRATCH/.keys/spack-push-key.gpg -``` - -To use the cache, pass the configuration as an option to `stack-config` via the `-c/--cache` flag: - -```bash -stack-config -b $build_path -r $recipe_path -s $system_config -c cache-config.yaml -``` - -??? warning "If you using an old binary build cache" - Since v3, Stackinator creates a sub-directory in the build cache for each mount point. - For example, in the above example, the build cache for the `/user-environment` mount point would be `$SCRATCH/uenv-cache/user-environment`. - The rationale for this is so that packages for different mount points are not mixed, to avoid having to relocate binaries. - - To continue using a build caches from before v3, first copy the `build_cache` path to a subdirectory, e.g.: - - ```bash - mkdir $SCRATCH/uenv-cache/user-environment - mv $SCRATCH/uenv-cache/build_cache $SCRATCH/uenv-cache/user-environment - ``` - -### Build-only caches - -A build cache can be configured to be read-only by not providing a `key` in the cache configuration file. - -## Creating a Build Cache - -To create a build cache we need two things: - -1. An empty directory where the cache will be populated by Spack. -2. A private PGP key - * Only required for Stackinator to push packages to the cache when it builds a package that was not in the cache. - -Creating the cache directory is easy! For example, to create a cache on your scratch storage: -```bash -mkdir $SCRATCH/uenv-cache -``` - -### Generating Keys - -An installation of Spack can be used to generate the key file: - -```bash -# create a key -spack gpg create - -# export key -spack gpg export --secret spack-push-key.gpg -``` - -See the [spack documentation](https://spack.readthedocs.io/en/latest/getting_started.html#gpg-signing) for more information about GPG keys. - -### Managing Keys - -The key needs to be in a location that is accessible during the build process, and secure. -To keep your PGP key secret, you can generate it then move it to a path with appropriate permissions. -In the example below, we create a path `.keys` for storing the key: -```bash -# create .keys path is visible only to you -mkdir $SCRATCH/.keys -chmod 700 $SCRATCH/.keys - -# generate the key -spack gpg create -spack gpg export --secret $SCRATCH/.keys/spack-push-key.gpg -chmod 600 $SCRATCH/.keys/spack-push-key.gpg -``` - -The cache-configuration would look like the following, where we assume that the cache is in `$SCRATCH/uenv-cache`: -```yaml -root: $SCRATCH/uenv-cache -key: $SCRATCH/.keys/spack-push-key.gpg -``` -!!! warning - Don't blindly copy this documentation's advice on security settings. - -!!! failure "Don't use `$HOME`" - Don't put the keys in `$HOME`, because the build process remounts `~` as a tmpfs, and you will get error messages that Spack can't read the key. - -## Force pushing to build cache - -When build caches are enabled, all packages in a each Spack environment are pushed to the build cache after the whole environment has been built successfully -- nothing will be pushed to the cache if there is an error when building one of the packages. - -When debugging a recipe, where failing builds have to be run multiple times, the overheads of rebuilding all packages from scratch can be wasteful. -To force push all packages that have been built, use the `cache-force` makefile target: - -```bash -env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make cache-force -``` diff --git a/docs/cluster-config.md b/docs/cluster-config.md index 2ce62019..cb6962f8 100644 --- a/docs/cluster-config.md +++ b/docs/cluster-config.md @@ -93,6 +93,13 @@ packages: version: ["git.59b6de6a91d9637809677c50cc48b607a91a9acb=main"] ``` +### Configuring Spack mirrors + +Mirrors and caches are **not** part of the system configuration. +They are supplied per-invocation with `stack-config --mirror `, because the locations involved (build caches, source caches) are often specific to the user running the build and may not be accessible to everyone using a system. + +See [Mirrors and Build Caches](build-caches.md) for the full reference and examples. + ## Site and System Configurations The `repo.yaml` configuration can be used to provide a list of additional Spack package repositories to use on the target system. diff --git a/docs/configuring.md b/docs/configuring.md index 099329b3..eea86c93 100644 --- a/docs/configuring.md +++ b/docs/configuring.md @@ -16,7 +16,8 @@ The following flags are required: The following flags are optional: -* `-c/--cache`: configure the [build cache](build-caches.md). +* `--mirror`: path to a [mirrors.yaml](build-caches.md) file configuring build caches and mirrors. +* `-c/--cache`: legacy build cache configuration file (deprecated; use `--mirror`). * `-m/--mount`: override the [mount point](installing.md) where the stack will be installed. * `--version`: print the stackinator version. * `-h/--help`: print help message. diff --git a/docs/mirrors.md b/docs/mirrors.md new file mode 100644 index 00000000..7115abb7 --- /dev/null +++ b/docs/mirrors.md @@ -0,0 +1,265 @@ +# Mirrors and Build Caches + +Spack can use *mirrors* and *caches* to speed up image builds and to build on systems with limited or no internet access. + +They are configured in a single YAML file passed to `stack-config` using the `--mirror` flag: + +```bash +stack-config -b $build -r $recipe -s $system --mirror mirrors.yaml +``` + +The file is not part of the [system configuration](cluster-config.md): mirror locations are usually specific to the person running the build, so each invocation provides its own. + +A `mirrors.yaml` can describe five kinds of entry, each optional and each documented below: + +| Entry | Count | Purpose | +|-------|-------|---------| +| [`buildcache`](#build-cache) | one | binary cache of built packages (the big build-time speed up) | +| [`bootstrap`](#bootstrap-mirror) | one | mirror used to bootstrap Spack itself | +| [`sourcemirror`](#source-mirrors) | many | read-only mirrors that provide package sources | +| [`sourcecache`](#source-cache) | one | writable local cache that fills with sources as you build | +| [`concretizer`](#concretizer-cache) | one | writable local cache that persists concretization results | + +!!! example + ```yaml title="mirrors.yaml" + buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: /capstor/scratch/bobsmith/.keys/spack-push-key.gpg + mount_specific: true + bootstrap: + url: https://bootstrap.spack.io + sourcemirror: + # more than one source mirror can be configred. + netmirror: + url: https://example.com/spack-sources + localmirror: + url: file://scratch/group15/spack-sources + sourcecache: + path: /capstor/scratch/bobsmith/spack-sources + concretizer: + path: /capstor/scratch/bobsmith/spack-concretizer + ``` + +!!! note + Paths inside the file (such as relative gpg key paths) are resolved relative to the directory containing the `mirrors.yaml`, so a self-contained mirror directory (the `mirrors.yaml` plus its keys) can be moved around freely. + + +To stop using any entry, remove (or comment out) it from `mirrors.yaml`. + +## Build cache + +A build cache is a binary cache of built packages. +Reusing binaries instead of rebuilding from source is roughly a 10x speed up — the difference between a 3 minute and a 30 minute image build — so a build cache is essential if you build images regularly. + +During a build Spack fetches packages from the cache when it can, and signs and pushes any package it has to build itself, so the cache improves over time. + +```yaml title="mirrors.yaml" +buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: /capstor/scratch/bobsmith/.keys/spack-push-key.gpg +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `url` | yes | location of the cache (a `file://` path, or an `http(s)://`, `s3://` or `oci://` URL) | +| `private_key` | no | PGP key used to sign and push packages (see [Keys](#keys)); omit for a read-only cache | +| `public_key` | no | PGP key used to verify downloaded packages | +| `name` | no | name Spack registers the mirror under (default `buildcache`) | +| `mount_specific` | no | store the cache in a per-mount-point sub-directory (default `false`) | + +### Read-only build cache + +Omit `private_key` to configure a read-only cache: Spack fetches packages from it but never signs or pushes anything back. +This is useful for consuming a shared team cache that you are not permitted to write to. + +```yaml title="mirrors.yaml" +buildcache: + url: file:///capstor/scratch/team/uenv-cache +``` + +### `mount_specific` + +Spack binaries embed the install prefix (the image's mount point), so binaries built for `/user-environment` cannot be reused at a different mount point. +Set `mount_specific: true` to append the mount point to the cache URL, giving each mount point its own sub-directory and avoiding relocation issues: + +```yaml +buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: /capstor/scratch/bobsmith/.keys/spack-push-key.gpg + mount_specific: true # packages stored under .../uenv-cache/user-environment +``` + +### Creating a build cache + +A build cache needs an empty directory and a PGP signing key: + +```bash +# 1. create the cache directory +mkdir -p /capstor/scratch/bobsmith/uenv-cache + +# 2. generate and export a signing key +spack gpg create +spack gpg export --secret /capstor/scratch/bobsmith/.keys/spack-push-key.gpg +``` + +See [Keys](#keys) for where to store the key. + +### Force pushing + +Packages are pushed to the cache after each environment builds successfully; nothing is pushed if a build fails. +When iterating on a recipe with failing builds, force-push everything built so far with the `cache-force` target: + +```bash +env --ignore-environment PATH=/usr/bin:/bin:`pwd -P`/spack/bin make cache-force +``` + +## Bootstrap mirror + +Spack bootstraps some of its own dependencies (such as the `clingo` concretizer) on first use. +A bootstrap mirror lets it do this without reaching the internet — useful on air-gapped systems. + +No key is needed: bootstrap binaries are verified by their sha256 sum, not by a GPG signature. + +The `url` can take two forms. + +**A local bootstrap mirror directory** (recommended) — a directory created with `spack bootstrap mirror`, which contains its own `metadata/sources` and `metadata/binaries` descriptors: + +```yaml title="mirrors.yaml" +bootstrap: + url: /capstor/scratch/team/bootstrap-mirror +``` + +Stackinator references the mirror's own metadata directly, so both source and binary bootstrapping work. Create one on a connected system and copy it across: + +```bash +spack bootstrap mirror --binary-packages /capstor/scratch/team/bootstrap-mirror +``` + +**A remote url** — `https://`, `s3://` or `oci://`: + +```yaml title="mirrors.yaml" +bootstrap: + url: https://bootstrap.example.com/mirror +``` + +A remote mirror supports **source** bootstrapping only; remote binary bootstrapping is not supported (it needs the per-package metadata that a local mirror directory provides). + +| Field | Required | Description | +|-------|----------|-------------| +| `url` | yes | a local `spack bootstrap mirror` directory, or a remote `https`/`s3`/`oci` url | + +## Source mirrors + +Source mirrors provide package **source** archives, and are read-only: Spack fetches sources from them but never writes to them. +Use them to build on air-gapped systems — populate a mirror on a connected system, mount it read-only, and Spack will fetch sources from it. +Any number of source mirrors can be listed; Spack searches them in order. + +```yaml title="mirrors.yaml" +sourcemirror: + internal: + url: https://mirror.example.com/spack-sources + scratch: + url: file:///capstor/scratch/team/spack-sources +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `url` | yes | location of the source mirror | + +Source mirrors need no keys: Spack verifies every downloaded source against the checksum in its package recipe, whether it comes from the upstream url or a mirror. + +Populate a source mirror on an internet-connected system with Spack: + +```bash +spack mirror create --directory /path/to/mirror --all +``` + +## Source cache + +A source cache is a single, **writable** local directory that Spack fills as it downloads sources. +On internet-connected systems Spack checks the cache first; on a miss it downloads the source and stores it, so later builds reuse it and download times shrink over time. + +Unlike a source mirror it is written to automatically, and is created on demand. + +```yaml title="mirrors.yaml" +sourcecache: + path: /capstor/scratch/bobsmith/spack-sources +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `path` | yes | absolute path to a local directory (environment variables are expanded) | + +## Concretizer cache + +The concretizer cache is a single, **writable** local directory in which Spack persists its **concretization results** — the output of concretizing a set of specs, so it does not have to be recomputed. +Concretization can be a large fraction of build time, so pointing this at a persistent location is worthwhile when build directories are ephemeral (e.g. created in `/dev/shm` and deleted after each build). + +```yaml title="mirrors.yaml" +concretizer: + path: /capstor/scratch/bobsmith/spack-concretizer +``` + +| Field | Required | Description | +|-------|----------|-------------| +| `path` | yes | absolute path to a local directory (environment variables are expanded) | + +This emits a `concretizer.yaml` that sets `concretizer:concretization_cache:{enable: true, url}`. +The cache is keyed by the hash of the solver inputs, so it can be reused safely across builds — stale entries simply miss. + +!!! info "concretizer cache is not a silver bullet" + The cache stores only the *result of the solve* for a given set of inputs. + Before it can be consulted, Spack still has to rebuild the full concretization problem on every run, loading the package recipes and enumerating the reusable packages, and that setup work is often the larger part of concretization. + So a cache hit skips the solver but not the setup: concretization gets faster, not free. + + The win is therefore largest for repeated builds of the same stack against a stable build cache (the previous solve is replayed), and smallest when the bulk of concretization time is in setup. + +!!! note "Requires Spack ≥ 1.1" + The `concretizer:concretization_cache` config key was introduced in Spack 1.1, and Spack 1.0 rejects it. + Stackinator infers the Spack version from the `spack.commit` in `config.yaml` (defaulting to a supported version when the commit is a branch or arbitrary SHA that cannot be pinned). + When it detects Spack 1.0 it skips the concretizer cache with a warning rather than producing a config that would fail the build. + +## Keys + +The build cache's `private_key` and `public_key` fields accept either: + +* a **path** — absolute, or relative to the directory containing `mirrors.yaml`; or +* a **base64-encoded key** inlined directly in `mirrors.yaml`. + +```yaml +buildcache: + url: file:///capstor/scratch/team/uenv-cache + private_key: $SCRATCH/.keys/spack-push-key.gpg # a path + public_key: mQINBGm4GvsBEACTyzQF...== # inline base64 +``` + +Generate a key with Spack, and keep the secret key somewhere private: + +```bash +mkdir $SCRATCH/.keys && chmod 700 $SCRATCH/.keys +spack gpg create +spack gpg export --secret $SCRATCH/.keys/spack-push-key.gpg +chmod 600 $SCRATCH/.keys/spack-push-key.gpg +``` + +See the [Spack documentation](https://spack.readthedocs.io/en/latest/getting_started.html#gpg-signing) for more on GPG keys. + +!!! warning "Don't use `$HOME`" + The build remounts `~` as a tmpfs, so keys under `$HOME` are not visible during the build and Spack will fail to read them. Use scratch storage instead. + +## Legacy `--cache` option + +Before `mirrors.yaml`, a build cache was configured with a separate `cache.yaml` file passed to `stack-config` via `-c/--cache`: + +```yaml title="cache.yaml" +root: $SCRATCH/uenv-cache +key: $SCRATCH/.keys/spack-push-key.gpg +``` + +```bash +stack-config -b $build -r $recipe -s $system -c cache.yaml +``` + +This is **deprecated** and equivalent to a single `buildcache` entry (with `mount_specific: true`). Prefer configuring the build cache in `mirrors.yaml`. + +Setting `key: null` configures a read-only cache that Spack fetches from but never pushes to. diff --git a/mkdocs.yml b/mkdocs.yml index 3ce996a0..b5d701cf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -8,7 +8,7 @@ nav: - 'Recipes': recipes.md - 'Cluster Configuration': cluster-config.md - 'Interfaces': interfaces.md - - 'Build Caches': build-caches.md + - 'Mirrors & Build Caches': mirrors.md - 'Spack 1.0': porting.md - 'Development': development.md # - Tutorial: tutorial.md diff --git a/pyproject.toml b/pyproject.toml index 583f5d46..671f8e63 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,9 +11,11 @@ license-files = ["LICENSE"] dynamic = ["version"] requires-python = ">=3.12" dependencies = [ + "python-magic", "Jinja2", "jsonschema", "PyYAML", + "boto3" ] classifiers = [ "Development Status :: 5 - Production/Stable", diff --git a/serve b/serve index 6a3328f3..40f3ad57 100755 --- a/serve +++ b/serve @@ -1,4 +1,4 @@ #!/usr/bin/env bash # use uv to run mkdocs with mkdocs-material and its dependencies installed -uv run --group docs mkdocs ${@:-serve} +uv run --group docs mkdocs ${@:-serve --livereload} diff --git a/stackinator/builder.py b/stackinator/builder.py index ae4997a1..feb9a312 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util +from . import VERSION, root_logger, spack_util def install(src, dst, *, ignore=None, symlinks=False): @@ -165,13 +165,16 @@ def environment_meta(self, recipe): self._environment_meta = meta def generate(self, recipe): + """Setup the recipe build environment.""" # make the paths, in case bwrap is not used, directly write to recipe.mount store_path = self.path / "store" if not recipe.no_bwrap else pathlib.Path(recipe.mount) tmp_path = self.path / "tmp" + config_path = self.path / "config" self.path.mkdir(exist_ok=True, parents=True) store_path.mkdir(exist_ok=True) tmp_path.mkdir(exist_ok=True) + config_path.mkdir(exist_ok=True) # check out the version of spack spack_version = recipe.spack_version @@ -221,18 +224,29 @@ def generate(self, recipe): lstrip_blocks=True, ) + # Write the spack mirror config artifacts (mirrors.yaml, bootstrap config, + # and the relocated gpg keys) into the config scope. These were fully + # resolved and validated by the recipe, so we just write the bytes. This + # must precede the Makefile render, which references the gpg key paths. + self._logger.debug(f"Writing the spack mirror configs to '{config_path}'") + for dest, content in recipe.mirrors.config_files(config_path).items(): + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(content) + # generate top level makefiles makefile_template = jinja_env.get_template("Makefile") with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache=recipe.mirror, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, + gpg_keys=recipe.mirrors.gpg_key_paths(config_path), + cache=recipe.build_cache_mirror, + buildcache_push=recipe.push_to_build_cache, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -300,8 +314,6 @@ def generate(self, recipe): # Generate the system configuration: the compilers, environments, etc. # that are defined for the target cluster. - config_path = self.path / "config" - config_path.mkdir(exist_ok=True) packages_path = config_path / "packages.yaml" # the packages.yaml configuration that will be used when building all environments @@ -312,13 +324,6 @@ def generate(self, recipe): with global_packages_path.open("w") as fid: fid.write(global_packages_yaml) - # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: - dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) - # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to # build the stack, and then be part of the upstream provided diff --git a/stackinator/cache.py b/stackinator/cache.py deleted file mode 100644 index 24177e33..00000000 --- a/stackinator/cache.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import pathlib - -import yaml - -from . import schema - - -def configuration_from_file(file, mount): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) - - # validate the yaml - schema.CacheValidator.validate(raw) - - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw - - -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } - } - } - - return yaml.dump(mirrors, default_flow_style=False) diff --git a/stackinator/main.py b/stackinator/main.py index 44406215..1ae6d617 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -74,6 +74,7 @@ def log_header(args): root_logger.info(f" system : {args.system}") mount = args.mount or "default" root_logger.info(f" mount : {mount}") + root_logger.info(f" mirror : {args.mirror}") root_logger.info(f" build cache: {args.cache}") root_logger.info(f" develop : {args.develop}") @@ -81,13 +82,37 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str) + parser.add_argument( + "-b", + "--build", + required=True, + type=str, + help="Where to set up the stackinator build directory. ('/tmp' is not allowed, use '/var/tmp')", + ) parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str) - parser.add_argument("-s", "--system", required=True, type=str) + parser.add_argument( + "-r", "--recipe", required=True, type=str, help="Name of (and/or path to) the Stackinator recipe." + ) + parser.add_argument( + "-s", "--system", required=True, type=str, help="Name of (and/or path to) the Stackinator system configuration." + ) parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str) - parser.add_argument("-c", "--cache", required=False, type=str) + parser.add_argument( + "-m", "--mount", required=False, type=str, help="The mount point where the environment will be located." + ) + parser.add_argument( + "--mirror", + required=False, + type=str, + help="Path to a mirrors.yaml file describing build caches and mirrors.", + ) + parser.add_argument( + "-c", + "--cache", + required=False, + type=str, + help="Legacy build cache configuration file (deprecated; use --mirror).", + ) parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py new file mode 100644 index 00000000..6c3c2ebb --- /dev/null +++ b/stackinator/mirror.py @@ -0,0 +1,500 @@ +from typing import Dict, List, Optional, Tuple +import base64 +import os +import pathlib +import urllib.parse +import yaml + +import magic + +from . import schema, root_logger + + +# GPG keys may be presented either ASCII-armored (these headers) or as binary +# data, in which case we fall back to libmagic to recognise the key. +ASCII_PGP_HEADERS = ( + b"-----BEGIN PGP PRIVATE KEY BLOCK-----", + b"-----BEGIN PGP PUBLIC KEY BLOCK-----", + b"-----BEGIN PGP MESSAGE-----", + b"-----BEGIN PGP SIGNATURE-----", +) + +# libmagic mime types that we accept as (binary) GPG key material. +GPG_KEY_MIME_TYPES = ( + "application/x-gnupg-keyring", + "application/pgp-keys", + "application/octet-stream", +) + + +def _supports_concretization_cache(spack_version: str) -> bool: + """Whether the given "major.minor" spack version supports the concretizer cache. + + The concretizer:concretization_cache config key was introduced in spack 1.1; + spack 1.0 rejects it (its concretizer schema forbids unknown keys). + """ + + major_minor = tuple(int(x) for x in spack_version.split(".")) + return major_minor >= (1, 1) + + +class MirrorError(RuntimeError): + """Exception class for errors thrown by mirror configuration problems.""" + + +class Mirrors: + """Fully validated and resolved definition of the spack mirrors for a recipe. + + The kinds of mirror have separate types: + + * buildcache - at most one, fetches and stores built packages (so it alone + has the mount_specific flag). With a private key it signs + and pushes packages too; without one it is read-only. None + if no build cache is configured. + * bootstrap - at most one, used to bootstrap spack itself (a local spack + bootstrap mirror directory, or a remote url). Needs no key + (bootstrap binaries are sha256-verified) and is emitted to + bootstrap.yaml, not the mirrors list. None if absent. + * source_mirrors - a name -> config mapping of any number of read-only source + mirrors (spack mirrors.yaml entries). They need no key: + sources are verified against the checksums in the package + recipes, never with gpg. + * source_cache - at most one, a writable local directory that spack fills as + it fetches sources (spack config:source_cache). None if + absent. This is not a mirror: it has no key and no url, and + is emitted to config.yaml rather than mirrors.yaml. + * concretizer_cache - at most one, a writable local directory persisting spack's + concretization results (concretizer:concretization_cache). + None if absent. Like source_cache it is not a mirror; it is + emitted to concretizer.yaml. It is only emitted for spack + >= 1.1 (spack 1.0 rejects the config key); when requested + against spack 1.0 it is skipped with a warning. + + All input processing - loading and schema-validating the system mirrors.yaml, + validating urls, and reading/decoding/validating gpg keys - happens eagerly in + the constructor, so any error is reported during recipe construction. Once + constructed, the object holds nothing but resolved, validated data, which is + presented to the builder as static artifacts via config_files() and + gpg_key_paths(). + """ + + KEY_STORE_DIR = "key_store" + MIRRORS_YAML = "mirrors.yaml" + CONFIG_YAML = "config.yaml" + BOOTSTRAP_YAML = "bootstrap.yaml" + CONCRETIZER_YAML = "concretizer.yaml" + + def __init__( + self, + system_config_root: pathlib.Path, + mount_path: pathlib.Path, + spack_version: str, + mirror_file: Optional[pathlib.Path] = None, + cmdline_cache: Optional[pathlib.Path] = None, + ): + """Load and fully resolve the mirror configuration. + + Mirrors are supplied with the --mirror command line option (mirror_file). + mount_path is the recipe mount point (used to make a build cache mount-specific). + spack_version is the best-effort "major.minor" spack version, used to gate the + concretizer cache (only emitted for spack >= 1.1). + cmdline_cache is an optional legacy cache.yaml passed on the command line (--cache). + + Relative paths in the mirror file (e.g. gpg keys) are resolved relative to the + directory containing the mirror file. + """ + + self._logger = root_logger + self._mount_path = mount_path + self._mirror_dir = mirror_file.parent if mirror_file is not None else None + + self.buildcache: Optional[Dict] = None + self.bootstrap: Optional[Dict] = None + self.source_mirrors: Dict[str, Dict] = {} + self.source_cache: Optional[Dict] = None + self.concretizer_cache: Optional[Dict] = None + + # The mirror configuration is supplied with --mirror, not the system + # configuration. Reject a mirrors.yaml in the system config so it is not + # silently ignored. + if (system_config_root / "mirrors.yaml").exists(): + raise MirrorError( + "A 'mirrors.yaml' in the system configuration is not supported.\n" + "Provide the mirror configuration with the '--mirror' command line option." + ) + + # Load and schema-validate the mirror file given on the command line. If none + # was given there are no mirrors; an empty config still validates (and picks up + # schema defaults such as an empty sourcemirror map). + if mirror_file is not None: + if not mirror_file.is_file(): + raise MirrorError(f"The mirror configuration file '{mirror_file}' does not exist.") + try: + with mirror_file.open() as fid: + raw_mirrors = yaml.load(fid, Loader=yaml.SafeLoader) + except (OSError, PermissionError) as err: + raise MirrorError(f"Could not open/read mirror file '{mirror_file}'.\n{err}") + else: + raw_mirrors = {} + + try: + schema.MirrorsValidator.validate(raw_mirrors) + except ValueError as err: + raise MirrorError(f"Mirror config does not comply with schema.\n{err}") + + # The build cache, if one is defined in mirrors.yaml. A build cache without + # a private_key is read-only: spack fetches from it but never pushes to it. + self.buildcache = raw_mirrors.get("buildcache") + + # A build cache passed via the deprecated cache.yaml file (the --cache CLI + # option) takes precedence over a buildcache defined in mirrors.yaml. + if cmdline_cache is not None: + if not cmdline_cache.is_file(): + raise MirrorError( + f"Binary cache configuration path given on the command line '{cmdline_cache}' does not exist." + ) + with cmdline_cache.open() as fid: + try: + raw_cache = yaml.load(fid, Loader=yaml.SafeLoader) + except ValueError as err: + raise MirrorError(f"Error loading yaml from cache config at '{cmdline_cache}'\n{err}") + try: + schema.CacheValidator.validate(raw_cache) + except ValueError as err: + raise MirrorError(f"Error validating contents of cache config at '{cmdline_cache}'.\n{err}") + + # a cache.yaml without a key configures a read-only (fetch-only) cache + self.buildcache = { + "name": "buildcache", + "url": raw_cache["root"], + "description": "Buildcache dest loaded from legacy cache.yaml", + "public_key": None, + "private_key": raw_cache.get("key"), + "mount_specific": True, + "cmdline": True, + } + self._logger.warning( + "Configuring the buildcache from the system cache.yaml file.\n" + "Please switch to using either the '--cache' option or the 'mirrors.yaml' file instead.\n" + f"The equivalent 'mirrors.yaml' would look like: \n" + f"{yaml.dump([self.buildcache], default_flow_style=False)}" + ) + + # The bootstrap mirror, the read-only source mirrors, the writable source + # cache, and the writable concretizer cache, if any are defined. + self.bootstrap = raw_mirrors.get("bootstrap") + self.source_mirrors = dict(raw_mirrors["sourcemirror"]) + self.source_cache = raw_mirrors.get("sourcecache") + self.concretizer_cache = raw_mirrors.get("concretizer") + + # Validate that every mirror url is well-formed (see _validate_url). + for name, mirror in self._iter_mirrors(): + self._validate_url(mirror["url"], name) + + # The source and concretizer caches are single writable local directories + # (spack config:source_cache / concretizer:concretization_cache), not mirrors: + # validate that each is an absolute path. Expand env vars now, because the + # build sandbox runs `env --ignore-environment` and so would not expand them + # at build time. + for cache_name, cache in (("source", self.source_cache), ("concretizer", self.concretizer_cache)): + if cache is not None: + path = os.path.expandvars(cache["path"]) + if not pathlib.Path(path).is_absolute(): + raise MirrorError(f"The {cache_name} cache path '{path}' is not absolute") + cache["path"] = path + + # The concretizer cache (concretizer:concretization_cache) was introduced in + # spack 1.1; spack 1.0 rejects the config key. Determine whether to emit it + # based on the best-effort spack version, and warn (not error) if it was + # requested but cannot be configured. + self._emit_concretizer_cache = self.concretizer_cache is not None and _supports_concretization_cache( + spack_version + ) + if self.concretizer_cache is not None and not self._emit_concretizer_cache: + self._logger.warning( + f"The concretizer cache is not supported by spack {spack_version} (requires >= 1.1) " + "and will not be configured." + ) + + # Resolve the bootstrap mirror. It is either a remote url, or a local spack + # bootstrap mirror directory (a `spack bootstrap mirror` output) whose own + # metadata/{sources,binaries} directories we reference directly. + self._bootstrap_remote = False + self._bootstrap_root: Optional[str] = None + self._bootstrap_metadata_dirs: List[str] = [] + if self.bootstrap is not None: + url = self.bootstrap["url"] + self._validate_url(url, "bootstrap") + if self._is_remote_url(url): + self._bootstrap_remote = True + else: + root = self._local_path(url) + if not root.is_dir(): + raise MirrorError(f"The bootstrap mirror directory '{root}' does not exist") + present = [sub for sub in ("sources", "binaries") if (root / "metadata" / sub).is_dir()] + if not present: + raise MirrorError( + f"The bootstrap mirror directory '{root}' has no 'metadata/sources' or " + f"'metadata/binaries' directory (is it a 'spack bootstrap mirror' output?)." + ) + self._bootstrap_root = root.as_posix() + self._bootstrap_metadata_dirs = present + + # Read, decode and validate every gpg key into memory. Each key is stored + # as (path-relative-to-config-root, raw bytes); the builder writes these + # verbatim into the build directory's key store. + key_store = pathlib.PurePosixPath(self.KEY_STORE_DIR) + self._key_files: List[Tuple[pathlib.PurePosixPath, bytes]] = [] + + # A build cache that pushes packages signs them with its private key. A build + # cache without a key is read-only, and is fetched from but never pushed to. + if self.buildcache is not None and self.buildcache["private_key"] is not None: + name = self.buildcache["name"] + self._key_files.append( + (key_store / f"{name}.priv.gpg", self._read_key(self.buildcache["private_key"], name)) + ) + + # The build cache may provide a public key, used to verify the packages + # fetched from it. It is the only mirror with keys at all: sources (and + # bootstrap binaries) are checksum-verified, and spack consults the gpg + # keyring only when verifying signed build-cache binaries. + if self.buildcache is not None and self.buildcache["public_key"] is not None: + name = self.buildcache["name"] + self._key_files.append((key_store / f"{name}.pub.gpg", self._read_key(self.buildcache["public_key"], name))) + + @property + def build_cache_mirror(self) -> Optional[str]: + """The build cache mirror name, or None if no build cache is configured. + + A build cache is fetched from (and its keys trusted) whether or not it has a + signing key; see push_to_build_cache for whether packages are pushed to it. + """ + + return self.buildcache["name"] if self.buildcache is not None else None + + @property + def push_to_build_cache(self) -> Optional[str]: + """The build cache mirror name to push built packages to, or None. + + Pushing requires a private signing key or an access pair for an S3 bucket; + a build cache configured without one is read-only - fetched from but never pushed to. + """ + + if self.buildcache is not None and self.buildcache["private_key"] is not None: + return self.buildcache["name"] + return None + + def _iter_mirrors(self): + """Yield (spack mirror name, config dict) for every real spack mirror. + + These are the entries written to the spack mirrors.yaml: the build cache + (whose name is the configurable 'name' field) and the source mirrors (named + by their key in mirrors.yaml). The bootstrap mirror is not a mirrors.yaml + entry, so it is handled separately. + """ + + if self.buildcache is not None: + yield self.buildcache["name"], self.buildcache + yield from self.source_mirrors.items() + + @staticmethod + def _is_remote_url(url: str) -> bool: + """True if url is a remote url (has a non-file scheme and a host).""" + + parsed = urllib.parse.urlparse(url) + return bool(parsed.scheme) and parsed.scheme != "file" and bool(parsed.netloc) + + @staticmethod + def _local_path(url: str) -> pathlib.Path: + """The local filesystem path for a file:// url or a bare path (env vars expanded).""" + + if url.startswith("file://"): + url = url[len("file://") :] + return pathlib.Path(os.path.expandvars(url)) + + def _validate_url(self, url: str, name: str): + """Validate that a mirror url is well-formed. + + Only the format of the url is checked: no attempt is made to connect to + remote mirrors, because a valid-but-unreachable url would otherwise block + until the network request times out. + """ + + if url.startswith("file://"): + # local mirror: verify that the root path is an existing directory + path = pathlib.Path(os.path.expandvars(url[len("file://") :])) + if not path.is_absolute(): + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not absolute") + if not path.is_dir(): + raise MirrorError(f"The mirror path '{path}' for mirror '{name}' is not a directory") + return + + parsed = urllib.parse.urlparse(url) + if not parsed.scheme: + # a bare path is accepted if absolute (e.g. the legacy command line cache) + if not pathlib.Path(url).is_absolute(): + raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url or absolute path") + elif not parsed.netloc: + # a remote mirror requires a well-formed url with both a scheme and a host + raise MirrorError(f"The mirror url '{url}' for mirror '{name}' is not a valid url") + + def _read_key(self, key: str, name: str) -> bytes: + """Resolve a key (a file path or base64 blob) to validated gpg key bytes. + + A key is either a path - absolute, or relative to the mirror file's directory + - or a base64-encoded blob inlined in the mirror file. The resulting bytes are + checked to be genuine gpg key material before being accepted. + """ + + # if it is a path (absolute, or relative to the mirror file), read it + path = pathlib.Path(os.path.expandvars(key)) + if not path.is_absolute(): + path = self._mirror_dir / path + + if path.is_file(): + binary_key = path.read_bytes() + else: + # otherwise it must be a base64-encoded key + try: + binary_key = base64.b64decode(key) + except ValueError: + raise MirrorError( + f"Key for mirror '{name}' is not valid: '{path}'. \n" + f"Must be a path to a GPG public key or a base64 encoded GPG public key. \n" + f"Check the key listed in mirrors.yaml in system config." + ) + + is_gpg_key = binary_key.startswith(ASCII_PGP_HEADERS) or ( + magic.from_buffer(binary_key, mime=True) in GPG_KEY_MIME_TYPES + ) + if not is_gpg_key: + raise MirrorError( + f"Key for mirror {name} is not a valid GPG key. \n" + f"The file (or base64) was readable, but the data itself was not a PGP key.\n" + f"Check the key listed in mirrors.yaml in system config." + ) + + return binary_key + + def gpg_key_paths(self, config_root: pathlib.Path) -> List[pathlib.Path]: + """The absolute paths the gpg keys are written to, for `spack gpg trust`.""" + + return [config_root / relpath for relpath, _ in self._key_files] + + def config_files(self, config_root: pathlib.Path) -> Dict[pathlib.Path, bytes]: + """The complete set of mirror config files to write under config_root. + + Returns a mapping of absolute file path -> file content. + """ + + files: Dict[pathlib.Path, bytes] = {} + + # the relocated gpg keys + for relpath, content in self._key_files: + files[config_root / relpath] = content + + # the spack mirrors.yaml + spack_mirrors: Dict[str, Dict] = {"mirrors": {}} + + if self.buildcache is not None: + url = self.buildcache["url"] + # a mount-specific build cache lives in a sub-directory named after the + # mount point: spack binaries embed the install prefix, so each mount + # point needs its own cache to avoid relocation issues. + if self.buildcache["mount_specific"]: + url = url.rstrip("/") + "/" + self._mount_path.as_posix().lstrip("/") + if self.buildcache["access_pair"] and self.buildcache["endpoint_url"]: + spack_mirrors["mirrors"][self.buildcache["name"]] = { + "fetch": { + "url": url, + "access_pair": self.buildcache["access_pair"], + "endpoint_url": self.buildcache["endpoint_url"], + }, + "push": { + "url": url, + "access_pair": self.buildcache["access_pair"], + "endpoint_url": self.buildcache["endpoint_url"], + }, + } + else: + spack_mirrors["mirrors"][self.buildcache["name"]] = { + "fetch": {"url": self.buildcache["url"]}, + "push": {"url": self.buildcache["url"]}, + } + + # source mirrors are read-only and provide sources only. Push url is required but never used + for name, mirror in self.source_mirrors.items(): + if mirror.get("access_pair") is not None and mirror.get("endpoint_url") is not None: + spack_mirrors["mirrors"][name] = { + "source": True, + "binary": False, + "fetch": { + "url": mirror["url"], + "access_pair": mirror["access_pair"], + "endpoint_url": mirror["endpoint_url"], + }, + "push": { + "url": mirror["url"], + "access_pair": mirror["access_pair"], + "endpoint_url": mirror["endpoint_url"], + }, + } + else: + spack_mirrors["mirrors"][name] = { + "source": True, + "binary": False, + "fetch": {"url": mirror["url"]}, + "push": {"url": mirror["url"]}, + } + + files[config_root / self.MIRRORS_YAML] = yaml.dump( + spack_mirrors, default_flow_style=False, sort_keys=False + ).encode() + + # the spack config.yaml setting the populate-as-you-go source cache (spack + # config:source_cache). + if self.source_cache is not None: + config_yaml = {"config": {"source_cache": self.source_cache["path"]}} + files[config_root / self.CONFIG_YAML] = yaml.dump(config_yaml, default_flow_style=False).encode() + + # the spack concretizer.yaml persisting concretization results. enable is set + # explicitly because it is opt-in in spack 1.1 (on by default only in >= 1.2). + # _emit_concretizer_cache gates this on the spack version (>= 1.1). + if self._emit_concretizer_cache: + concretizer_yaml = { + "concretizer": { + "concretization_cache": { + "enable": True, + "url": self.concretizer_cache["path"], + } + } + } + files[config_root / self.CONCRETIZER_YAML] = yaml.dump(concretizer_yaml, default_flow_style=False).encode() + + # the spack bootstrap.yaml, if a bootstrap mirror is set. Bootstrapping reads + # bootstrap:sources (not the mirrors list), and each source's `metadata` is a + # directory describing it. No gpg key is involved (bootstrap binaries are + # sha256-verified). + if self.bootstrap is not None: + sources = [] + trusted = {} + if self._bootstrap_remote: + # a remote mirror: generate a local source descriptor pointing at it. + # this covers source bootstrapping; remote binary bootstrapping (which + # needs per-package sha256 metadata) is not supported. + metadata_dir = config_root / "bootstrap" / "bootstrap-mirror" + metadata_yaml = {"type": "install", "info": {"url": self.bootstrap["url"]}} + files[metadata_dir / "metadata.yaml"] = yaml.dump(metadata_yaml, default_flow_style=False).encode() + sources.append({"name": "bootstrap-mirror", "metadata": str(metadata_dir)}) + trusted["bootstrap-mirror"] = True + else: + # a local spack bootstrap mirror: reference its own metadata directories. + for sub in self._bootstrap_metadata_dirs: + name = f"bootstrap-{sub}" + sources.append({"name": name, "metadata": f"{self._bootstrap_root}/metadata/{sub}"}) + trusted[name] = True + + bootstrap_yaml = {"bootstrap": {"sources": sources, "trusted": trusted}} + files[config_root / self.BOOTSTRAP_YAML] = yaml.dump(bootstrap_yaml, default_flow_style=False).encode() + + return files diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 30bf8b58..7a2b1ded 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -5,7 +5,7 @@ import jinja2 import yaml -from . import cache, root_logger, schema, spack_util +from . import root_logger, schema, spack_util, mirror from .etc.envvars import EnvVarSet @@ -176,15 +176,22 @@ def __init__(self, args): ) raise RuntimeError("Ivalid default-view in the recipe.") - # optional mirror configurtion - mirrors_path = self.path / "mirrors.yaml" - if mirrors_path.is_file(): - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") + # determine the version of spack being used: it is inferred (best effort) + # from the spack commit in config.yaml, defaulting to the latest supported + # version when it cannot be determined. This must precede the mirror + # configuration, which gates the concretizer cache on the spack version. + self.spack_version = self.find_spack_version(args.develop) - self.mirror = (args.cache, self.mount) + # resolve the mirror configuration provided with --mirror. --cache is the + # legacy path. + self._logger.debug("Configuring mirrors.") + self.mirrors = mirror.Mirrors( + self.system_config_path, + self.mount, + self.spack_version, + pathlib.Path(args.mirror) if args.mirror else None, + pathlib.Path(args.cache) if args.cache else None, + ) # optional post install hook if self.post_install_hook is not None: @@ -198,11 +205,6 @@ def __init__(self, args): else: self._logger.debug("no pre install hook provided") - # determine the version of spack being used: - # currently this just returns 1.0... develop is ignored - # --develop flag will imply the next release of spack after 1.0 is supported properly - self.spack_version = self.find_spack_version(args.develop) - # Returns: # Path: if the recipe contains a spack package repository # None: if there is the recipe contains no repo @@ -213,6 +215,20 @@ def spack_repo(self): return repo_path return None + # Returns: + # Path: if the recipe specified a build cache mirror + # None: if no build cache mirror is used + @property + def build_cache_mirror(self): + return self.mirrors.build_cache_mirror + + # Returns: + # str: the build cache mirror name to push built packages to + # None: if there is no build cache, or it is read-only (no signing key) + @property + def push_to_build_cache(self): + return self.mirrors.push_to_build_cache + # Returns: # Path: of the recipe extra path if it exists # None: if there is no user-provided extra path in the recipe @@ -243,32 +259,6 @@ def pre_install_hook(self): return hook_path return None - # Returns a dictionary with the following fields - # - # root: /path/to/cache - # path: /path/to/cache/user-environment - # key: /path/to/private-pgp-key - @property - def mirror(self): - return self._mirror - - # configuration is a tuple with two fields: - # - a Path of the yaml file containing the cache configuration - # - the mount point of the image - @mirror.setter - def mirror(self, configuration): - self._logger.debug(f"configuring build cache mirror with {configuration}") - self._mirror = None - - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - - self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - @property def config(self): return self._config @@ -288,10 +278,30 @@ def config(self, config_path): def with_modules(self) -> bool: return self.modules is not None - # In Stackinator 6 we replaced logic required to determine the - # pre 1.0 Spack version. + # Make a best-effort determination of the "major.minor" version of spack being + # used, inferred from the spack commit in config.yaml. This is only a hint: the + # commit can be an arbitrary branch/tag/sha, so when the version cannot be pinned + # we default to the latest supported version ("1.1"). Returns a "major.minor" + # string (e.g. "1.0", "1.1"). def find_spack_version(self, develop): - return "1.0" + # the latest supported version, used when the version cannot be determined + # (an explicit --develop, the default branch, or an unrecognised commit). + default = "1.1" + + if develop: + return default + + commit = self.config["spack"]["commit"] + if commit is None or commit in ("develop", "main"): + return default + + # match a release branch/tag (releases/v1.0, v1.1, v1.1.2) or a bare "1.0", + # and extract the major.minor version. + match = re.search(r"v?(\d+)\.(\d+)(?:\.\d+)?", commit) + if match: + return f"{match.group(1)}.{match.group(2)}" + + return default @property def default_view(self): @@ -523,10 +533,9 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None files["makefile"] = makefile_template.render( compilers=self.compilers, - push_to_cache=push_to_cache, + buildcache=self.push_to_build_cache, spack_version=self.spack_version, ) @@ -554,10 +563,9 @@ def environment_files(self): jenv.filters["py2yaml"] = schema.py2yaml makefile_template = jenv.get_template("Makefile.environments") - push_to_cache = self.mirror is not None files["makefile"] = makefile_template.render( environments=self.environments, - push_to_cache=push_to_cache, + buildcache=self.push_to_build_cache, spack_version=self.spack_version, ) diff --git a/stackinator/schema.py b/stackinator/schema.py index 3a2a9842..d461ff0e 100644 --- a/stackinator/schema.py +++ b/stackinator/schema.py @@ -121,3 +121,4 @@ def check_module_paths(instance): EnvironmentsValidator = SchemaValidator(prefix / "schema/environments.json") CacheValidator = SchemaValidator(prefix / "schema/cache.json") ModulesValidator = SchemaValidator(prefix / "schema/modules.json", check_module_paths) +MirrorsValidator = SchemaValidator(prefix / "schema/mirror.json") diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json new file mode 100644 index 00000000..61af5d6a --- /dev/null +++ b/stackinator/schema/mirror.json @@ -0,0 +1,83 @@ +{ + "type" : "object", + "additionalProperties": false, + "properties": { + "bootstrap": { + "type": "object", + "properties": { + "description": {"type": "string", "default": ""}, + "url": {"type": "string"} + }, + "additionalProperties": false, + "required": ["url"] + }, + "buildcache": { + "type": "object", + "properties": { + "name": { + "type": "string", + "default": "buildcache" + }, + "description": {"type": "string", "default": ""}, + "url": {"type": "string"}, + "public_key": {"type": ["string", "null"], "default": null}, + "private_key": {"type": ["string", "null"], "default": null}, + "mount_specific": { + "type": "boolean", + "default": false + }, + "cmdline": { + "type": "boolean", + "default": false + }, + "access_pair": { + "type": "array", + "items": { "type": "string" }, + "minItems": 2, + "maxItems": 2 + }, + "endpoint_url": {"type": "string"} + }, + "additionalProperties": false, + "required": ["url"] + }, + "sourcemirror": { + "type": "object", + "default": {}, + "additionalProperties": { + "type": "object", + "properties": { + "description": {"type": "string", "default": ""}, + "url": {"type": "string"}, + "access_pair": { + "type": "array", + "items": { "type": "string" }, + "minItems": 2, + "maxItems": 2 + }, + "endpoint_url": {"type": "string"} + }, + "additionalProperties": false, + "required": ["url"] + } + }, + "sourcecache": { + "type": "object", + "properties": { + "description": {"type": "string", "default": ""}, + "path": {"type": "string"} + }, + "additionalProperties": false, + "required": ["path"] + }, + "concretizer": { + "type": "object", + "properties": { + "description": {"type": "string", "default": ""}, + "path": {"type": "string"} + }, + "additionalProperties": false, + "required": ["path"] + } + } +} \ No newline at end of file diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea58..25c47e15 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -19,8 +19,8 @@ spack-setup: spack-version printf "spack version... "; \ version="$$($(SANDBOX) $(SPACK) --version)"; \ printf "%s\n" "$$version"; \ - printf "checking if spack concretizer works... "; \ - $(SANDBOX) $(SPACK_HELPER) -d spec zlib > $(BUILD_ROOT)/spack-bootstrap-output 2>&1; \ + printf "bootstrapping spack... "; \ + $(SANDBOX) $(SPACK_HELPER) -d bootstrap now > $(BUILD_ROOT)/spack-bootstrap-output 2>&1; \ if [ "$$?" != "0" ]; then \ printf " failed, see %s\n" $(BUILD_ROOT)/spack-bootstrap-output; \ exit 1; \ @@ -34,11 +34,15 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% if cache %} + @echo "Pulling and trusting keys from configured buildcaches." $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} {% endif %} + @echo "Adding mirror gpg keys." + {% for key_path in gpg_keys %} + $(SANDBOX) $(SPACK) gpg trust {{ key_path }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list touch mirror-setup compilers: mirror-setup @@ -77,14 +81,14 @@ store.squashfs: post-install # Force push all built packages to the build cache cache-force: mirror-setup -{% if cache.key %} +{% if buildcache_push %} $(warning ================================================================================) $(warning Generate the config in order to force push partially built compiler environments) $(warning if this step is performed with partially built compiler envs, you will) $(warning likely have to start a fresh build (but that's okay, because build caches FTW)) $(warning ================================================================================) $(SANDBOX) $(MAKE) -C generate-config - $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --rebuild-index --only=package alpscache \ + $(SANDBOX) $(SPACK) --color=never -C $(STORE)/config buildcache create --only=package {{ buildcache_push }} \ $$($(SANDBOX) $(SPACK_HELPER) -C $(STORE)/config find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ diff --git a/stackinator/templates/Makefile.compilers b/stackinator/templates/Makefile.compilers index 8418fbf0..0447863f 100644 --- a/stackinator/templates/Makefile.compilers +++ b/stackinator/templates/Makefile.compilers @@ -18,8 +18,8 @@ all:{% for compiler in compilers %} {{ compiler }}/generated/build_cache{% endfo {% for compiler, config in compilers.items() %} {{ compiler }}/generated/build_cache: {{ compiler }}/generated/env -{% if push_to_cache %} - $(SPACK) -e ./{{ compiler }} buildcache create --rebuild-index --only=package alpscache \ +{% if buildcache %} + $(SPACK) -e ./{{ compiler }} buildcache create --only=package {{ buildcache }} \ $$($(SPACK_HELPER) -e ./{{ compiler }} find --format '{name};{/hash}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | cut -d ';' -f2) diff --git a/stackinator/templates/Makefile.environments b/stackinator/templates/Makefile.environments index 5a530232..74699d46 100644 --- a/stackinator/templates/Makefile.environments +++ b/stackinator/templates/Makefile.environments @@ -17,8 +17,8 @@ all:{% for env in environments %} {{ env }}/generated/build_cache{% endfor %} # Push built packages to a binary cache if a key has been provided {% for env, config in environments.items() %} {{ env }}/generated/build_cache: {{ env }}/generated/view_config -{% if push_to_cache %} - $(SPACK) --color=never -e ./{{ env }} buildcache create --rebuild-index --only=package alpscache \ +{% if buildcache %} + $(SPACK) --color=never -e ./{{ env }} buildcache create --only=package {{ buildcache }} \ $$($(SPACK_HELPER) -e ./{{ env }} find --format '{name};{/hash};version={version}' \ | grep -v -E '^({% for p in config.exclude_from_cache %}{{ pipejoiner() }}{{ p }}{% endfor %});'\ | grep -v -E 'version=git\.'\ diff --git a/unittests/__init__.py b/unittests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/unittests/data/systems/mirror-bad-concretizer/mirrors.yaml b/unittests/data/systems/mirror-bad-concretizer/mirrors.yaml new file mode 100644 index 00000000..4f982e05 --- /dev/null +++ b/unittests/data/systems/mirror-bad-concretizer/mirrors.yaml @@ -0,0 +1,2 @@ +concretizer: + path: relative/not/absolute diff --git a/unittests/data/systems/mirror-bad-key/bad_key.gpg b/unittests/data/systems/mirror-bad-key/bad_key.gpg new file mode 100644 index 00000000..d7980bbf --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/bad_key.gpg @@ -0,0 +1 @@ +This is a bad key \ No newline at end of file diff --git a/unittests/data/systems/mirror-bad-key/mirrors.yaml b/unittests/data/systems/mirror-bad-key/mirrors.yaml new file mode 100644 index 00000000..a3d01283 --- /dev/null +++ b/unittests/data/systems/mirror-bad-key/mirrors.yaml @@ -0,0 +1,3 @@ +buildcache: + url: https://mirror.spack.io + private_key: bad_key.gpg diff --git a/unittests/data/systems/mirror-bad-keypath/mirrors.yaml b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml new file mode 100644 index 00000000..5ac6af9e --- /dev/null +++ b/unittests/data/systems/mirror-bad-keypath/mirrors.yaml @@ -0,0 +1,3 @@ +buildcache: + url: https://mirror.spack.io + private_key: /path/doesnt/exist diff --git a/unittests/data/systems/mirror-bad-sourcecache/mirrors.yaml b/unittests/data/systems/mirror-bad-sourcecache/mirrors.yaml new file mode 100644 index 00000000..dd59d5c1 --- /dev/null +++ b/unittests/data/systems/mirror-bad-sourcecache/mirrors.yaml @@ -0,0 +1,2 @@ +sourcecache: + path: relative/not/absolute diff --git a/unittests/data/systems/mirror-bad-url/mirrors.yaml b/unittests/data/systems/mirror-bad-url/mirrors.yaml new file mode 100644 index 00000000..61341619 --- /dev/null +++ b/unittests/data/systems/mirror-bad-url/mirrors.yaml @@ -0,0 +1,3 @@ +sourcemirror: + bad-url: + url: not-a-valid-url \ No newline at end of file diff --git a/unittests/data/systems/mirror-no-sourcecache/mirrors.yaml b/unittests/data/systems/mirror-no-sourcecache/mirrors.yaml new file mode 100644 index 00000000..7a1ecdff --- /dev/null +++ b/unittests/data/systems/mirror-no-sourcecache/mirrors.yaml @@ -0,0 +1,5 @@ +bootstrap: + url: https://mirror.spack.io +sourcemirror: + mirror1: + url: https://github.com diff --git a/unittests/data/systems/mirror-ok/cache-nokey.yaml b/unittests/data/systems/mirror-ok/cache-nokey.yaml new file mode 100644 index 00000000..5fcd561b --- /dev/null +++ b/unittests/data/systems/mirror-ok/cache-nokey.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: null diff --git a/unittests/data/systems/mirror-ok/cache.yaml b/unittests/data/systems/mirror-ok/cache.yaml new file mode 100644 index 00000000..ad7de37c --- /dev/null +++ b/unittests/data/systems/mirror-ok/cache.yaml @@ -0,0 +1,2 @@ +root: /tmp/foo +key: ../../test-gpg-priv.asc diff --git a/unittests/data/systems/mirror-ok/mirrors.yaml b/unittests/data/systems/mirror-ok/mirrors.yaml new file mode 100644 index 00000000..a55004f3 --- /dev/null +++ b/unittests/data/systems/mirror-ok/mirrors.yaml @@ -0,0 +1,57 @@ +bootstrap: + url: https://mirror.spack.io +buildcache: + url: https://mirror.spack.io + public_key: "\ + mQINBGm4GvsBEACTyzQFPfRUgo1Wmb9/KgrSr/EFVobRX3LlrcAMemo4nFRdS88aCcEhRWzYQ8ML\ + eGvcxFbzbAoEZACpThMspYOwFsVzIUc3lYQT7g9M/KNEPHztqaTWCqESYmzDFtaLYys6AQP52KMB\ + 2x0ya8NNd9whd2a9Vc3yD3u9qW8iqkqxDjNYtTc9Lo7T8DHlhJ79TKm8f3w0QZowTxPYh5NA8GiF\ + kBN6J8hmg39LekC1kAWi2BwjDzRll19zVQtYfv+b4i/ripqmMNp4zcU93Ox1ReUFOmO3eiIq3GSK\ + IbXw7h/NGLAImIeuMzce5cqz65iVk7+iyKeXtx5dSUGskiLR2voX8xVOGVhNtxfiolviyUAhT4kb\ + WcWK6ipZ9h/nEyeZ9GYtoDkKMguet2a4bJCBsQmo4FR9Pf5c7qqs79obxLXzZe9zDnj1sbxAabJh\ + jLUdwJqIPKvdPNy3F3nOBeKY8Jf1D+Y3szxzJX8gwqrNSyCbZUeXsaQhMZWUVoztxUXW+qfC8jlA\ + TfoUQuQPC9Zr+8wU/3uUlKtChQo0prgwHAEHezwNpyEhZZc884RIt55DNKllH9UeqNwUWKpZYHG3\ + qVxV+oi7MenLeKvfsg6nVCL0CETtB88dOen7BFfBZj9NRszRqIlVudDFf+5gqKQ0f1H241w/n4nH\ + KEAzm7kma4cV8QARAQABtBtUZXN0IEtleSA8bm9wZUBub3doZXJlLmNvbT6JAlEEEwEKADsWIQSe\ + CE84hgwq8uroW+w4Qgxu0DsXiAUCabga+wIbAwULCQgHAgIiAgYVCgkICwIEFgIDAQIeBwIXgAAK\ + CRA4Qgxu0DsXiBbaD/4w7MlAf5SuOxrbH3fruN7k8NPxjwsvocB3VGo5AxzIy18C78IOYm5F2EQ+\ + LpjsK05t1ZPsHFsBaLduo0CODcF8m1gJLI8S0uMmVUywnDGJjMKYiZNeIqXDlohVciD2KPIxWL+i\ + qgho1BhFKlnQkXgEaxotUXFwHiKqcBwFcq21nu3PRVqsobMTk/UgeJhoSZf7ZIj5SyVHa6YVCoMr\ + HB0TqRz3xIW/TDl+oxyfEdhMZ8iU6IdohfMkCP+ayZEpdx1SS37S47SRxbgNblbfJ9PfJD/dQzx1\ + WnGVXxinSPjTTxIwNCyiEIrxqvLnk+O8fUHeEIrSqn9P0bZrCv02gAPUIfl7l4gN4boSgWEsfS5i\ + /KL8ZUDGZCb9Fux64zaM17lHfwGWCAKbi1KjYRL4W7zaVmps6MfdLOcJdQSdpQufs9vRbMhiDW1x\ + glsvz8JzPYiF2xRqJsx2odiufx4Mrrq5yxER07sDjKzUZYF6xD8qC5BAh6/xDUE+p8EOqc0HsqTE\ + PhqBLdqGLf8GaXj3I9F6ZCH5dtSmehB6Q77KJ1hWTm9OzDdYm2apExbMIB9Z2H5c8FLZfIb4lnpW\ + lhtP97eAix9JnBzoTe8QeaV5hcvQoqypTu5rD3ne2kQHlavmSeq5KVVWrIKGsfnZNRqDg7vKg/fw\ + kx1f2T8qMLVfGxogJrkCDQRpuBr7ARAA08MhBUQkj4nVaLmW+Xa5e+nTnE8nYoKPYJIpbIDYP2Pe\ + mD9Yca9HGWHnzUgsH4KEvdETrT/Uj9i3o+6xNZJ1Xz0GQkKW+2Vttl5ZBKwipHsn2iP+e78wAhwm\ + HqYZi/ymgLJDEXmrgyXNr+cKaAI2cEWOb+Vomgu+WFF4ENG/UiIJH6zP5JY0TcSC1Ao47y4qL6bH\ + Mfzz2zi3JpbvOi1/uY2TvMgitaL45tsukuEYMFfEEnd4Vyl0LSCFYv3zQx5JRSbu5ujlRdHnopS0\ + UeWZMV+iEXYZ6wKVFtQ6nvxxDpjMf8k3Q5Ss+z0F9oTRwdlaJhdhXZx8PleBu0Ah1Wxd9+V6tYgF\ + zrjsJB9eaXKIEeioZi8xNFB6RYUuNgjIfTtgtps685LeiGCy5q55MkC2FuKVAdv+YZcuJNdy/3gx\ + Kg5bLz/aVQRfxakDQHI7kp9fl3bDK5jbWeY8EKvtTI8x7fxthPZP6Az2g4zp+ZojgEUdUXveuw0Y\ + 3MVaMu0ehG81nUHNU1tu7ELuhDWM6VkigokS1zptBsPbKojfp/oZJn6DGD1LZ+QyIdSUkjyfcs9H\ + sfuyAbrgzVCmbcNY42x3IOZZZjIfQ66bJZpGht6kHEfO896oB2d+a7KS25ZWa5G+SsWntv5nnclr\ + 6DYFz3ThuYVmjQDLh8jN78/f45Jd6N8AEQEAAYkCNgQYAQoAIBYhBJ4ITziGDCry6uhb7DhCDG7Q\ + OxeIBQJpuBr7AhsMAAoJEDhCDG7QOxeIVx8P/1wxrmmYWhIMDObXIpCM3vxq8dO+84nTuBQbomKR\ + NURKOiCwgndEL3N38pf0gAToSIatrTF2VdkJsksyMEIzUaNmsYiHA9xYqhmCJ2pIqWeh2ONsNdmw\ + Fg/M5mwZpvwl28Z2MpJP+NY6u52a3jxkxpGY1Q4+KxgMRhqXe6faXQtYwwUiYVGPSznQPudYLZ2Z\ + +b8rGrz0AUVvvSWt3bVbUwZIMVSK0RVIWxG/sW7dWhkhtev+04fUlaxHnQ2b8G3h6AjONmLcIlpx\ + 7p1dVvolEqV0YQUgosl47J3tLnzacsqNzIS1Dya0ukLrXAYmeQzQWvwhLpLqjMh3cqLl5SkjatB7\ + xU9Qu4IXENnvWSnqCRZzz6CbU/81FopTGgJfxbYok2v78O5qTdkbeszSHN8uCuvhpPKruHZgsFc6\ + lw+hYhtB8YXbB8lT2f1Fp0DeEnPM+OzRgjeRYl3gmE8/1PtKGuTCOJzTxTtLWorFYtV0DXiOq4Vd\ + eYkR+m3vNiYVkdALN5uIL8goYrPvs/fvq1wI49iyKw6B3pE5xIQSEjgPpwJ/7hvQUhenJTtrNRs8\ + eKXSnjHZjhJbgIReoXSQwG44RqNtiV8dJsdPu98P27keSigBB5kguB0gCWeFVHkLfpBR3aRxSacG\ + gllMF++N8+7T4/ehkA/hs2udYRkSCANLQ3I3" + private_key: ../../test-gpg-priv.asc + mount_specific: false + cmdline: false +sourcemirror: + mirror1: + url: https://github.com + mirror2: + url: https://github.com/spack +sourcecache: + path: /scratch/spack-sources +concretizer: + path: /scratch/spack-concretizer \ No newline at end of file diff --git a/unittests/data/systems/mirror-readonly-cache/mirrors.yaml b/unittests/data/systems/mirror-readonly-cache/mirrors.yaml new file mode 100644 index 00000000..0c041531 --- /dev/null +++ b/unittests/data/systems/mirror-readonly-cache/mirrors.yaml @@ -0,0 +1,2 @@ +buildcache: + url: https://mirror.spack.io diff --git a/unittests/data/test-gpg-priv.asc b/unittests/data/test-gpg-priv.asc new file mode 100644 index 00000000..eaa2dc19 Binary files /dev/null and b/unittests/data/test-gpg-priv.asc differ diff --git a/unittests/data/test-gpg-pub.asc b/unittests/data/test-gpg-pub.asc new file mode 100644 index 00000000..aa72b028 Binary files /dev/null and b/unittests/data/test-gpg-pub.asc differ diff --git a/unittests/test_mirrors.py b/unittests/test_mirrors.py new file mode 100644 index 00000000..2aca5a6a --- /dev/null +++ b/unittests/test_mirrors.py @@ -0,0 +1,446 @@ +import base64 +import pathlib +import pytest +import stackinator.mirror as mirror +import yaml + + +@pytest.fixture +def test_path(): + return pathlib.Path(__file__).parent.resolve() + + +@pytest.fixture +def systems_path(test_path): + return test_path / "data" / "systems" + + +@pytest.fixture +def mount_path(): + return pathlib.Path("/user-environment") + + +@pytest.fixture +def clean_root(tmp_path): + """A system config directory with no mirrors.yaml (which the constructor rejects).""" + root = tmp_path / "system-config" + root.mkdir() + return root + + +@pytest.fixture +def mirror_ok(systems_path): + """The path to the well-formed mirror file, as it would be passed to --mirror.""" + return systems_path / "mirror-ok" / "mirrors.yaml" + + +def test_mirror_init(clean_root, mount_path, systems_path, mirror_ok): + """Check that the three kinds of mirror are resolved into separate members.""" + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) + + with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: + pub_key_b64 = base64.b64encode(pub_key_file.read()).decode() + + # the build cache, with its schema-defaulted name, flags and (empty) fields + assert mirrors_obj.buildcache == { + "name": "buildcache", + "description": "", + "url": "https://mirror.spack.io", + "private_key": "../../test-gpg-priv.asc", + "public_key": pub_key_b64, + "mount_specific": False, + "cmdline": False, + } + + assert mirrors_obj.bootstrap == { + "url": "https://mirror.spack.io", + "description": "", + } + + # non-required fields are always present, defaulted to "" by the schema + assert mirrors_obj.source_mirrors == { + "mirror1": {"url": "https://github.com", "description": ""}, + "mirror2": {"url": "https://github.com/spack", "description": ""}, + } + + # the writable, populate-as-you-go source cache + assert mirrors_obj.source_cache == {"path": "/scratch/spack-sources", "description": ""} + + # the writable concretizer cache (persists concretization results) + assert mirrors_obj.concretizer_cache == {"path": "/scratch/spack-concretizer", "description": ""} + + # the build cache mirror name is derived from the build cache's 'name' field + assert mirrors_obj.build_cache_mirror == "buildcache" + + +def test_system_mirrors_yaml_rejected(systems_path, mount_path): + """A mirrors.yaml in the system configuration is not supported and is rejected.""" + + # mirror-ok contains a mirrors.yaml; passing it as the system config root must raise. + with pytest.raises(mirror.MirrorError): + mirror.Mirrors(systems_path / "mirror-ok", mount_path, "1.1") + + +def test_missing_mirror_file(clean_root, mount_path): + """A --mirror file that does not exist raises MirrorError.""" + + with pytest.raises(mirror.MirrorError): + mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=clean_root / "does-not-exist.yaml") + + +def test_no_mirror_file(clean_root, mount_path): + """With no --mirror file there are no mirrors configured.""" + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1") + + assert mirrors_obj.buildcache is None + assert mirrors_obj.bootstrap is None + assert mirrors_obj.source_cache is None + assert mirrors_obj.source_mirrors == {} + assert mirrors_obj.build_cache_mirror is None + + +def test_mirror_init_bad_url(clean_root, mount_path, systems_path): + """Check that MirrorError is raised for a bad url.""" + + with pytest.raises(mirror.MirrorError): + mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=systems_path / "mirror-bad-url/mirrors.yaml") + + +def test_command_line_cache(clean_root, mount_path, systems_path, mirror_ok): + """Check that adding a cache from the command line works.""" + + mirrors = mirror.Mirrors( + clean_root, mount_path, "1.1", mirror_file=mirror_ok, cmdline_cache=systems_path / "mirror-ok/cache.yaml" + ) + + # the command line cache overrides any build cache defined in the mirror file, + # and is named "buildcache" like any other build cache. + assert mirrors.build_cache_mirror == "buildcache" + assert mirrors.buildcache["name"] == "buildcache" + assert mirrors.buildcache["url"] == "/tmp/foo" + assert mirrors.buildcache["cmdline"] + assert mirrors.buildcache["mount_specific"] + + # it has a signing key, so it is pushed to + assert mirrors.push_to_build_cache == "buildcache" + + # the bootstrap and source mirrors from the mirror file are still present + assert mirrors.bootstrap is not None + assert set(mirrors.source_mirrors) == {"mirror1", "mirror2"} + + +def test_keyless_command_line_cache(tmp_path, clean_root, mount_path, systems_path, mirror_ok): + """A cache.yaml without a key configures a read-only (fetch-only) build cache.""" + + mirrors = mirror.Mirrors( + clean_root, mount_path, "1.1", mirror_file=mirror_ok, cmdline_cache=systems_path / "mirror-ok/cache-nokey.yaml" + ) + + # the cache exists (so it is fetched from), but has no signing key ... + assert mirrors.build_cache_mirror == "buildcache" + assert mirrors.buildcache["private_key"] is None + + # ... so it is never pushed to + assert mirrors.push_to_build_cache is None + + files = mirrors.config_files(tmp_path) + + # no private key is written + assert tmp_path / "key_store" / "buildcache.priv.gpg" not in files + + # the mirror is emitted with a fetch url but no push url + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + assert data["mirrors"]["buildcache"] == {"fetch": {"url": "/tmp/foo/user-environment"}} + + +def test_readonly_buildcache(tmp_path, clean_root, mount_path, systems_path): + """A buildcache in the mirror file without a private_key is read-only (fetch-only).""" + + mirrors = mirror.Mirrors( + clean_root, mount_path, "1.1", mirror_file=systems_path / "mirror-readonly-cache/mirrors.yaml" + ) + + # the cache exists (so it is fetched from), but has no signing key ... + assert mirrors.build_cache_mirror == "buildcache" + assert mirrors.buildcache["private_key"] is None + + # ... so it is never pushed to + assert mirrors.push_to_build_cache is None + + files = mirrors.config_files(tmp_path) + + # no private key is written + assert tmp_path / "key_store" / "buildcache.priv.gpg" not in files + + # the mirror is emitted with a fetch url but no push url + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + assert data["mirrors"]["buildcache"] == {"fetch": {"url": "https://mirror.spack.io"}} + + +def test_config_files(tmp_path, clean_root, mount_path, mirror_ok): + """Check that config_files presents the complete set of mirror config artifacts. + + The mirror file lives in a different directory from the (empty) system config root, + so this also exercises relative key paths resolving against the mirror file's dir. + """ + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) + files = mirrors_obj.config_files(tmp_path) + + expected = { + tmp_path / "mirrors.yaml", + tmp_path / "config.yaml", + tmp_path / "concretizer.yaml", + tmp_path / "bootstrap.yaml", + tmp_path / "bootstrap" / "bootstrap-mirror" / "metadata.yaml", + tmp_path / "key_store" / "buildcache.priv.gpg", + tmp_path / "key_store" / "buildcache.pub.gpg", + } + assert set(files.keys()) == expected + + # every artifact is presented as raw bytes, ready to be written verbatim + assert all(isinstance(content, bytes) for content in files.values()) + + +def test_spack_mirrors_yaml(tmp_path, clean_root, mount_path, mirror_ok): + """Check that the mirrors.yaml passed to spack is correct""" + + valid_spack_yaml = { + "mirrors": { + "buildcache": { + "fetch": {"url": "https://mirror.spack.io"}, + "push": {"url": "https://mirror.spack.io"}, + }, + "mirror1": { + "source": True, + "binary": False, + "fetch": {"url": "https://github.com"}, + }, + "mirror2": { + "source": True, + "binary": False, + "fetch": {"url": "https://github.com/spack"}, + }, + } + } + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) + files = mirrors_obj.config_files(tmp_path) + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + + assert data == valid_spack_yaml + + +def test_mount_specific_buildcache(tmp_path, clean_root, mount_path, mirror_ok): + """A mount_specific buildcache should have the mount point appended to its url. + + Spack binaries embed the install prefix (the mount point), so a mount_specific + cache is namespaced per-mount-point to avoid relocation issues / collisions. + """ + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) + + # mirror-ok's buildcache is mount_specific: false by default; enable it. + mirrors_obj.buildcache["mount_specific"] = True + + files = mirrors_obj.config_files(tmp_path) + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + + # the buildcache url gains the mount point as a sub-directory ... + assert data["mirrors"]["buildcache"]["fetch"]["url"] == "https://mirror.spack.io/user-environment" + assert data["mirrors"]["buildcache"]["push"]["url"] == "https://mirror.spack.io/user-environment" + + # ... while other mirrors are left untouched. + assert data["mirrors"]["mirror1"]["fetch"]["url"] == "https://github.com" + + +def test_mount_specific_disabled(tmp_path, clean_root, mount_path, mirror_ok): + """A buildcache with mount_specific false is unchanged, even when a mount point is set.""" + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) + + # confirm the fixture leaves the flag off + assert mirrors_obj.buildcache["mount_specific"] is False + + files = mirrors_obj.config_files(tmp_path) + data = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + + assert data["mirrors"]["buildcache"]["fetch"]["url"] == "https://mirror.spack.io" + + +def test_remote_bootstrap_configs(tmp_path, clean_root, mount_path, mirror_ok): + """A remote bootstrap url generates a local source descriptor pointing at it.""" + + valid_yaml = { + "bootstrap": { + "sources": [ + { + "name": "bootstrap-mirror", + "metadata": str(tmp_path / "bootstrap/bootstrap-mirror"), + } + ], + "trusted": {"bootstrap-mirror": True}, + } + } + valid_metadata = { + "type": "install", + "info": { + "url": "https://mirror.spack.io", + }, + } + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) + files = mirrors_obj.config_files(tmp_path) + + bs_data = yaml.safe_load(files[tmp_path / "bootstrap.yaml"]) + assert bs_data == valid_yaml + + metadata = yaml.safe_load(files[tmp_path / "bootstrap/bootstrap-mirror/metadata.yaml"]) + assert metadata == valid_metadata + + # the bootstrap mirror is not added to the spack mirrors list + mirrors = yaml.safe_load(files[tmp_path / "mirrors.yaml"]) + assert "bootstrap" not in mirrors["mirrors"] + + +def test_local_bootstrap_configs(tmp_path, clean_root, mount_path): + """A local bootstrap mirror dir is referenced via its own metadata directories.""" + + # fake a `spack bootstrap mirror` output directory + boot = tmp_path / "bootstrap-mirror" + (boot / "metadata" / "sources").mkdir(parents=True) + (boot / "metadata" / "binaries").mkdir(parents=True) + + mirror_file = tmp_path / "mirrors.yaml" + mirror_file.write_text(f"bootstrap:\n url: {boot}\n") + + config_root = tmp_path / "config" + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_file) + files = mirrors_obj.config_files(config_root) + + bs_data = yaml.safe_load(files[config_root / "bootstrap.yaml"]) + assert bs_data == { + "bootstrap": { + "sources": [ + {"name": "bootstrap-sources", "metadata": f"{boot}/metadata/sources"}, + {"name": "bootstrap-binaries", "metadata": f"{boot}/metadata/binaries"}, + ], + "trusted": {"bootstrap-sources": True, "bootstrap-binaries": True}, + } + } + + # no metadata is generated (it lives in the mirror), and nothing in the mirrors list + assert not any(p.name == "metadata.yaml" for p in files) + mirrors = yaml.safe_load(files[config_root / "mirrors.yaml"]) + assert "bootstrap" not in mirrors["mirrors"] + + +def test_local_bootstrap_missing_metadata(tmp_path, clean_root, mount_path): + """A local bootstrap dir without metadata/sources|binaries is rejected.""" + + boot = tmp_path / "not-a-bootstrap-mirror" + boot.mkdir() + + mirror_file = tmp_path / "mirrors.yaml" + mirror_file.write_text(f"bootstrap:\n url: {boot}\n") + + with pytest.raises(mirror.MirrorError): + mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_file) + + +def test_keys(tmp_path, clean_root, mount_path, systems_path, mirror_ok): + """Check that gpg keys are decoded, relocated and reported consistently.""" + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) + files = mirrors_obj.config_files(tmp_path) + + # the buildcache is the only mirror with keys (sources are checksum-verified) + pub_files = {p for p in files if p.name.endswith(".pub.gpg")} + assert {p.name for p in pub_files} == {"buildcache.pub.gpg"} + + # the buildcache public key is inlined base64 in the fixture; the decoded bytes + # must match the key file it was encoded from + with (systems_path / "../test-gpg-pub.asc").open("rb") as pub_key_file: + assert files[tmp_path / "key_store/buildcache.pub.gpg"] == pub_key_file.read() + + # gpg_key_paths reports exactly the key files that config_files writes + key_files = {p for p in files if p.parent.name == "key_store"} + assert set(mirrors_obj.gpg_key_paths(tmp_path)) == key_files + + +def test_local_caches_config(tmp_path, clean_root, mount_path, mirror_ok): + """The source cache is emitted to config.yaml; the concretizer cache to concretizer.yaml.""" + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_ok) + files = mirrors_obj.config_files(tmp_path) + + config_data = yaml.safe_load(files[tmp_path / "config.yaml"]) + assert config_data == {"config": {"source_cache": "/scratch/spack-sources"}} + + concretizer_data = yaml.safe_load(files[tmp_path / "concretizer.yaml"]) + assert concretizer_data == { + "concretizer": {"concretization_cache": {"enable": True, "url": "/scratch/spack-concretizer"}} + } + + +def test_concretizer_cache_only(tmp_path, clean_root, mount_path): + """A mirror file with a concretizer cache but no source cache emits only concretizer.yaml.""" + + mirror_file = tmp_path / "mirrors.yaml" + mirror_file.write_text("concretizer:\n path: /scratch/only-concretizer\n") + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=mirror_file) + files = mirrors_obj.config_files(tmp_path) + + assert mirrors_obj.source_cache is None + assert tmp_path / "config.yaml" not in files + concretizer_data = yaml.safe_load(files[tmp_path / "concretizer.yaml"]) + assert concretizer_data == { + "concretizer": {"concretization_cache": {"enable": True, "url": "/scratch/only-concretizer"}} + } + + +def test_concretizer_cache_skipped_on_spack_1_0(tmp_path, clean_root, mount_path): + """The concretizer cache is not emitted for spack 1.0 (which rejects the config key).""" + + mirror_file = tmp_path / "mirrors.yaml" + mirror_file.write_text("concretizer:\n path: /scratch/only-concretizer\n") + + mirrors_obj = mirror.Mirrors(clean_root, mount_path, "1.0", mirror_file=mirror_file) + files = mirrors_obj.config_files(tmp_path) + + # the cache is still recorded as requested, but no concretizer.yaml is written + assert mirrors_obj.concretizer_cache is not None + assert tmp_path / "concretizer.yaml" not in files + + +def test_local_caches_absent(tmp_path, clean_root, mount_path, systems_path): + """No config.yaml or concretizer.yaml is generated when neither cache is configured.""" + + # mirror-no-sourcecache has neither a sourcecache nor a concretizer entry + mirrors_obj = mirror.Mirrors( + clean_root, mount_path, "1.1", mirror_file=systems_path / "mirror-no-sourcecache/mirrors.yaml" + ) + files = mirrors_obj.config_files(tmp_path) + + assert mirrors_obj.source_cache is None + assert mirrors_obj.concretizer_cache is None + assert tmp_path / "config.yaml" not in files + assert tmp_path / "concretizer.yaml" not in files + + +@pytest.mark.parametrize( + "system_name", + [ + "mirror-bad-key", + "mirror-bad-keypath", + "mirror-bad-sourcecache", + "mirror-bad-concretizer", + ], +) +def test_bad_config(clean_root, mount_path, systems_path, system_name): + """Check that MirrorError is raised at construction for bad keys or a bad cache path.""" + + with pytest.raises(mirror.MirrorError): + mirror.Mirrors(clean_root, mount_path, "1.1", mirror_file=systems_path / system_name / "mirrors.yaml") diff --git a/unittests/test_recipe.py b/unittests/test_recipe.py new file mode 100644 index 00000000..7aa1fabe --- /dev/null +++ b/unittests/test_recipe.py @@ -0,0 +1,39 @@ +import pytest + +from stackinator.recipe import Recipe + + +def make_recipe(commit): + """A Recipe with only the spack config populated, bypassing the heavy __init__.""" + recipe = Recipe.__new__(Recipe) + recipe._config = {"spack": {"commit": commit}} + return recipe + + +@pytest.mark.parametrize( + "commit, expected", + [ + # release branches and tags pin the major.minor version + ("releases/v1.0", "1.0"), + ("releases/v1.1", "1.1"), + ("releases/v2.0", "2.0"), + ("v1.0.0", "1.0"), + ("v1.1.3", "1.1"), + ("1.0", "1.0"), + # the version cannot be determined -> default to the latest supported (1.1) + ("develop", "1.1"), + ("main", "1.1"), + (None, "1.1"), + ("a3f9c1e8b2d4f6a7c9e1b3d5f7a9c1e3b5d7f9a1", "1.1"), + ], +) +def test_find_spack_version(commit, expected): + """find_spack_version infers major.minor from the commit, defaulting to 1.1.""" + recipe = make_recipe(commit) + assert recipe.find_spack_version(develop=False) == expected + + +def test_find_spack_version_develop_flag(): + """The --develop flag forces the latest supported version regardless of commit.""" + recipe = make_recipe("releases/v1.0") + assert recipe.find_spack_version(develop=True) == "1.1"