Add Elixir and OTP CI coverage and stabilize distributed tests

.formatter.exs

@@ -1,4 +1,9 @@
 # Used by "mix format"
 [
-  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+  inputs: [
+    "{mix,.formatter}.exs",
+    "{config,lib,test}/**/*.{ex,exs}",
+    "priv/ci/*.exs",
+    "priv/bench/comparison.exs"
+  ]
 ]

.github/workflows/campaigns.yml

@@ -0,0 +1,91 @@
+name: Fault campaigns
+
+on:
+  schedule:
+    - cron: "23 3 * * *"
+  workflow_dispatch:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: false
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  MIX_ENV: test
+  ERL_FLAGS: "+S 4:4"
+  GROUP_HISTORY_STEPS: "2000"
+  GROUP_TEST_DIAGNOSTICS: ${{ github.workspace }}/_build/test-diagnostics
+  ERL_CRASH_DUMP: ${{ github.workspace }}/_build/test-diagnostics/erl_crash.dump
+
+jobs:
+  histories-and-faults:
+    name: Seed ${{ matrix.seed }} / ${{ matrix.schedulers }} peer schedulers
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        seed: [104729, 130363, 155921]
+        schedulers: ["1", "2", "4"]
+    env:
+      GROUP_PEER_SCHEDULERS: ${{ matrix.schedulers }}
+      CAMPAIGN_SEED: ${{ matrix.seed }}
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
+        with:
+          elixir-version: "> 0"
+          otp-version: "> 0"
+      - run: mix deps.get
+      - name: Repeat histories, partitions, churn, restarts and race regressions
+        run: |
+          mkdir -p "$GROUP_TEST_DIAGNOSTICS"
+          mix compile --warnings-as-errors
+          for iteration in 0 1 2 3 4; do
+            seed=$((CAMPAIGN_SEED + iteration))
+            mix test --warnings-as-errors --seed "$seed" 2>&1 |
+              tee "$GROUP_TEST_DIAGNOSTICS/run-$seed.log"
+          done
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: campaign-${{ matrix.seed }}-${{ matrix.schedulers }}
+          path: _build/test-diagnostics
+          retention-days: 14
+
+  mutations:
+    name: Contract mutations
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    env:
+      GROUP_HISTORY_STEPS: "100"
+      GROUP_PEER_SCHEDULERS: "2"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
+        with:
+          elixir-version: "> 0"
+          otp-version: "> 0"
+      - run: mix deps.get
+      - name: Require behavioral regressions to be detected
+        run: elixir priv/ci/mutations.exs
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: mutation-reports
+          path: |
+            _build/mutations/**/*.log
+            _build/mutations/**/diagnostics
+          retention-days: 14

.github/workflows/ci.yml

@@ -0,0 +1,105 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+  release:
+    types: [published]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  MIX_ENV: test
+  ERL_FLAGS: "+S 4:4"
+  GROUP_PEER_SCHEDULERS: "2"
+  GROUP_TEST_DIAGNOSTICS: ${{ github.workspace }}/_build/test-diagnostics
+  ERL_CRASH_DUMP: ${{ github.workspace }}/_build/test-diagnostics/erl_crash.dump
+
+jobs:
+  local:
+    name: Local checks
+    runs-on: ubuntu-24.04
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+      - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
+        with:
+          elixir-version: "> 0"
+          otp-version: "> 0"
+      - run: mix deps.get
+      - name: Formatting, compiler checks, local examples and histories
+        run: |
+          mkdir -p "$GROUP_TEST_DIAGNOSTICS"
+          mix format --check-formatted 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/format.log"
+          mix compile --warnings-as-errors 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/compile.log"
+          mix test --only local --warnings-as-errors 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/local.log"
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: failure()
+        with:
+          name: local-failure
+          path: _build/test-diagnostics
+          retention-days: 14
+
+  test:
+    name: ${{ matrix.latest && 'Latest stable Elixir / OTP' || format('Elixir {0} / OTP {1}', matrix.elixir, matrix.otp) }}
+    needs: local
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - elixir: "1.19"
+            otp: "26"
+          - elixir: "1.19"
+            otp: "27"
+          - elixir: "1.19"
+            otp: "28"
+          - elixir: "1.20"
+            otp: "27"
+          - elixir: "1.20"
+            otp: "28"
+          # Unlike "latest", these ranges exclude prereleases.
+          - elixir: "> 0"
+            otp: "> 0"
+            latest: true
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
+        with:
+          elixir-version: ${{ matrix.elixir }}
+          otp-version: ${{ matrix.otp }}
+
+      - name: Install dependencies
+        run: mix deps.get
+
+      - name: Compile with warnings as errors
+        run: |
+          mkdir -p "$GROUP_TEST_DIAGNOSTICS"
+          mix compile --warnings-as-errors 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/compile.log"
+
+      - name: Run local and distributed tests
+        run: mix test --warnings-as-errors 2>&1 | tee "$GROUP_TEST_DIAGNOSTICS/test.log"
+
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: failure()
+        with:
+          name: runtime-failure-${{ strategy.job-index }}
+          path: _build/test-diagnostics
+          retention-days: 14

.github/workflows/performance.yml

@@ -0,0 +1,83 @@
+name: Performance comparison
+
+on:
+  workflow_dispatch:
+    inputs:
+      baseline:
+        description: "Baseline ref (use a previous release tag for release review)"
+        type: string
+        default: main
+        required: true
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  compare:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    env:
+      ERL_FLAGS: "+S 4:4"
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          path: candidate
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Select baseline
+        id: baseline
+        env:
+          BASELINE: ${{ inputs.baseline }}
+        run: |
+          if [ -z "$BASELINE" ]; then
+            BASELINE=$(git -C candidate describe --tags --abbrev=0 "${GITHUB_SHA}^")
+          fi
+          printf 'ref=%s\n' "$BASELINE" >> "$GITHUB_OUTPUT"
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          path: baseline
+          ref: ${{ steps.baseline.outputs.ref }}
+          persist-credentials: false
+      - uses: erlef/setup-beam@54075bcc5e249e4758d363f27d099f55d843f124 # v1.24.1
+        with:
+          elixir-version: "> 0"
+          otp-version: "> 0"
+      - name: Record environment and revisions
+        run: |
+          mkdir -p reports
+          {
+            elixir --version
+            uname -a
+            lscpu
+            git -C baseline rev-parse HEAD
+            git -C candidate rev-parse HEAD
+          } | tee reports/environment.log
+      - name: Benchee baseline and candidate on the same runner
+        run: |
+          GROUP_BENCH_ROOT="$PWD/baseline" \
+          GROUP_BENCH_OUTPUT="$PWD/reports/baseline.benchee" \
+          GROUP_BENCH_TAG=baseline \
+            elixir candidate/priv/bench/comparison.exs 2>&1 | tee reports/benchee-baseline.log
+          GROUP_BENCH_ROOT="$PWD/candidate" \
+          GROUP_BENCH_OUTPUT="$PWD/reports/candidate.benchee" \
+          GROUP_BENCH_BASELINE="$PWD/reports/baseline.benchee" \
+            elixir candidate/priv/bench/comparison.exs 2>&1 | tee reports/benchee-candidate.log
+      - name: Distributed load and recovery, baseline then candidate
+        run: |
+          epmd -daemon
+          for revision in baseline candidate; do
+            timeout 20m bash "$revision/priv/bench/run_distributed.sh" 2>&1 |
+              tee "reports/distributed-$revision.log"
+          done
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        if: always()
+        with:
+          name: performance-comparison
+          path: reports
+          retention-days: 30

README.md

@@ -400,6 +400,28 @@ mix test
 See [`test/README.md`](test/README.md) for details on the distributed test
 infrastructure.
 
+GitHub Actions runs fast local checks first, then the full suite on pull requests,
+pushes to `main`, and releases using Elixir 1.19 / OTP 26–28, Elixir 1.20 / OTP 27–28, and the latest
+stable Elixir / OTP pair. Version ranges pick up new patch releases automatically;
+the latest-stable job also picks up new minor and major releases, excluding
+prereleases. Every job treats compilation and test warnings as errors. Nightly
+campaigns vary seeds and peer scheduler counts, run larger generated histories,
+and check targeted mutations. Performance comparisons run separately.
+See [`test/README.md`](test/README.md#ci-lanes) for lane budgets and release review.
+
+To run the same checks locally:
+
+```bash
+export MIX_ENV=test
+export ERL_FLAGS="+S 4:4"
+export GROUP_PEER_SCHEDULERS=2
+mix deps.get
+mix format --check-formatted
+mix compile --warnings-as-errors
+mix test --only local --warnings-as-errors # no distribution startup
+mix test --warnings-as-errors
+```
+
 ## Benchmarks
 
 ```bash

priv/bench/comparison.exs

@@ -0,0 +1,56 @@
+# Use the same harness and Benchee version for both revisions. Only the Group
+# path changes; this also works when the baseline predates this harness.
+root = System.get_env("GROUP_BENCH_ROOT") || Path.expand("../..", __DIR__)
+Mix.install([{:group, path: root}, {:benchee, "== 1.5.1"}])
+
+{:ok, supervisor} = Group.start_link(name: :comparison, shards: 8, log: false)
+parent = self()
+
+members =
+  for _ <- 1..128 do
+    spawn_link(fn ->
+      :ok = Group.join(:comparison, "members", %{})
+      send(parent, {:ready, self()})
+
+      receive do
+        :stop -> :ok
+      end
+    end)
+  end
+
+for pid <- members do
+  receive do
+    {:ready, ^pid} -> :ok
+  after
+    5_000 -> raise "benchmark member did not become ready"
+  end
+end
+
+:ok = Group.register(:comparison, "lookup", %{value: 1})
+output = System.get_env("GROUP_BENCH_OUTPUT", "comparison.benchee")
+baseline = System.get_env("GROUP_BENCH_BASELINE")
+
+try do
+  Benchee.run(
+    %{
+      "lookup" => fn -> Group.lookup(:comparison, "lookup") end,
+      "members/128" => fn -> Group.members(:comparison, "members") end,
+      "register/unregister" => fn ->
+        :ok = Group.register(:comparison, "registry-cycle", %{})
+        :ok = Group.unregister(:comparison, "registry-cycle")
+      end,
+      "join/leave" => fn ->
+        :ok = Group.join(:comparison, "pg-cycle", %{})
+        :ok = Group.leave(:comparison, "pg-cycle")
+      end
+    },
+    time: 3,
+    warmup: 1,
+    memory_time: 1,
+    save: [path: output, tag: System.get_env("GROUP_BENCH_TAG", "candidate")],
+    load: if(baseline, do: [baseline], else: [])
+  )
+after
+  for pid <- members, do: send(pid, :stop)
+  Supervisor.stop(supervisor)
+end