diff --git a/.github/workflows/qemu-image-build.yml b/.github/workflows/qemu-image-build.yml index af3e2026f6..1e55d07347 100644 --- a/.github/workflows/qemu-image-build.yml +++ b/.github/workflows/qemu-image-build.yml @@ -11,6 +11,10 @@ on: description: Upload the built disk image as an artifact for the release workflow type: boolean default: false + test_pg_upgrade: + description: Run tests/pg_upgrade/qemu.sh against the built image + type: boolean + default: false secrets: DEV_AWS_ROLE: description: AWS role for pushing the stamp to the Nix binary cache (optional) @@ -34,6 +38,10 @@ on: options: - arm64 - amd64 + test_pg_upgrade: + description: Run tests/pg_upgrade/qemu.sh against the built image + type: boolean + default: false permissions: contents: read @@ -80,6 +88,7 @@ jobs: GIT_SHA: ${{ github.sha }} PACKAGES_GIT_SHA: ${{ steps.resolve-git-sha.outputs.sha }} PUBLISH: ${{ inputs.publish }} + TEST: ${{ inputs.test_pg_upgrade }} run: | set -euo pipefail case "$TARGET_ARCH" in @@ -90,7 +99,7 @@ jobs: stamp=$(nix eval --raw ".#packages.$system.qemu-image-stamp-$POSTGRES_MAJOR_VERSION") echo "system=$system" >>"$GITHUB_OUTPUT" cached=false - if [[ $PUBLISH != true && $PACKAGES_GIT_SHA == "$GIT_SHA" ]] && nix path-info --store "$CACHE_URL" "$stamp" &>/dev/null; then + if [[ $PUBLISH != true && $TEST != true && $PACKAGES_GIT_SHA == "$GIT_SHA" ]] && nix path-info --store "$CACHE_URL" "$stamp" &>/dev/null; then echo "::notice::image already passed ($stamp), skipping build" cached=true fi @@ -126,6 +135,13 @@ jobs: GIT_SHA: ${{ steps.resolve-git-sha.outputs.sha }} run: BUILD_QEMU_IMAGE_HW_VIRT_ONLY=1 nix run .#build-qemu-image "$POSTGRES_MAJOR_VERSION" "$TARGET_ARCH" + - name: Test pg_upgrade + if: steps.stamp.outputs.cached != 'true' && inputs.test_pg_upgrade + run: | + IMAGE=$(echo packer-work-qemu-*/output-cloudimg/packer-cloudimg) \ + PG_UPGRADE_STORE_PATH=$(nix build --no-link --print-out-paths ".#psql_$POSTGRES_MAJOR_VERSION/bin") \ + nix shell --inputs-from . nixpkgs#qemu nixpkgs#cloud-utils nixpkgs#sshpass --command tests/pg_upgrade/qemu.sh + - name: Grab release version id: process_release_version run: | diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 69cf423475..0df420a2e1 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -414,7 +414,9 @@ EXTRA_NIX_CONF CATALOG_LOCAL="/tmp/nix-catalog-${NIX_FLAKE_VERSION}-psql_${PGVERSION}-${SYSTEM}.json" log "Fetching catalog from: $CATALOG_S3" - if ! aws s3 cp "$CATALOG_S3" "$CATALOG_LOCAL" --region ap-southeast-1; then + if [ -n "${PG_UPGRADE_STORE_PATH:-}" ]; then + jq -n --arg s "$SYSTEM" --arg p "$PG_UPGRADE_STORE_PATH" '{($s): $p}' >"$CATALOG_LOCAL" + elif ! aws s3 cp "$CATALOG_S3" "$CATALOG_LOCAL" --region ap-southeast-1; then log "ERROR: Failed to fetch catalog from $CATALOG_S3" false # not exit: let the ERR trap run cleanup fi diff --git a/tests/pg_upgrade/qemu.sh b/tests/pg_upgrade/qemu.sh new file mode 100755 index 0000000000..efb7d05878 --- /dev/null +++ b/tests/pg_upgrade/qemu.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash + +set -euo pipefail + +: "${IMAGE:?}" "${PG_UPGRADE_STORE_PATH:?}" + +cd "$(git rev-parse --show-toplevel)" +[[ $(uname -s) == Darwin ]] && accel=hvf || accel=kvm +work=$(mktemp -d) +trap '[ ! -f "$work/pid" ] || kill "$(cat "$work/pid")"; rm -rf "$work"' EXIT + +share=$(dirname "$(dirname "$(command -v qemu-system-aarch64)")")/share/qemu +install -m 644 "$share/edk2-arm-vars.fd" "$work/vars.fd" +qemu-img create -q -f qcow2 -F qcow2 -b "$(realpath "$IMAGE")" "$work/disk.qcow2" 30G +cloud-localds "$work/seed.iso" nix/packages/build-qemu-image/user-data-cloudimg nix/packages/build-qemu-image/meta-data + +qemu-system-aarch64 \ + -machine "virt,gic-version=max,highmem=on,accel=$accel" -cpu host -smp 4 -m 8192 \ + -display none -serial "file:${CONSOLE_LOG:-/tmp/pg-upgrade-qemu-console.log}" -daemonize -pidfile "$work/pid" \ + -drive "if=pflash,format=raw,readonly=on,file=$share/edk2-aarch64-code.fd" \ + -drive "if=pflash,format=raw,file=$work/vars.fd" \ + -drive "file=$work/disk.qcow2,if=virtio,format=qcow2" \ + -drive "file=$work/seed.iso,format=raw,if=virtio" \ + -netdev "user,id=n0,hostfwd=tcp:127.0.0.1:2222-:22" -device virtio-net-pci,netdev=n0 + +export SSHPASS=ubuntu +ssh_opts=(-p 2222 -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR) +vm() { sshpass -e ssh "${ssh_opts[@]}" ubuntu@127.0.0.1 "$@"; } + +# nix copy needs a trusted user over key auth, which only root is +ssh-keygen -q -N '' -f "$work/key" +until vm 'sudo install -d -m 700 /root/.ssh && sudo tee /root/.ssh/authorized_keys >/dev/null' <"$work/key.pub" 2>/dev/null; do sleep 5; done +vm() { ssh "${ssh_opts[@]}" -i "$work/key" root@127.0.0.1 "$@"; } +NIX_SSHOPTS="${ssh_opts[*]} -i $work/key" nix copy --no-check-sigs \ + --to "ssh-ng://root@127.0.0.1?remote-program=/nix/var/nix/profiles/default/bin/nix-daemon" \ + "$PG_UPGRADE_STORE_PATH" + +tar --no-xattrs -c migrations/db ansible/files/admin_api_scripts/pg_upgrade_scripts ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql ansible/files/stat_extension.sql tests/pg_upgrade/tests | + vm 'rm -rf /tmp/t && mkdir /tmp/t && tar -C /tmp/t -x' + +vm bash -eu -s "$PG_UPGRADE_STORE_PATH" <<'EOF' +export PATH=/usr/lib/postgresql/bin:$PATH PGDATA=/var/lib/postgresql/data PG_MAJOR_VERSION=17 +export PG_UPGRADE_STORE_PATH=$1 +cd /tmp/t +scripts=$PWD/ansible/files/admin_api_scripts/pg_upgrade_scripts +psql_admin() { psql -h localhost -U supabase_admin -d postgres -v ON_ERROR_STOP=1 -X "$@"; } + +# initdb's bootstrap child reads timezone data from the compiled-in /usr/share/postgresql +ln -sfn /usr/lib/postgresql/share/postgresql /usr/share/postgresql +install -d -o postgres -g postgres -m 0750 /data/pgdata +flags=(--data-checksums) +[ "$(pg_config --version | grep -o '[0-9]*' | head -1)" -ge 17 ] && flags+=(--locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8) +su postgres -c "LC_ALL=en_US.UTF-8 LOCALE_ARCHIVE=/usr/lib/locale/locale-archive pg_ctl -D /var/lib/postgresql/data initdb -o '--allow-group-access --username=supabase_admin ${flags[*]}'" +bash "$scripts/pgsodium_getkey.sh" >/dev/null +systemctl start postgresql +until pg_isready -h localhost; do sleep 1; done + +psql_admin -c "create role postgres superuser login; alter database postgres owner to postgres;" +for f in pgbouncer_config/pgbouncer_auth_schema.sql stat_extension.sql; do psql_admin -U postgres -f "ansible/files/$f"; done +POSTGRES_HOST=localhost sh migrations/db/migrate.sh +for f in tests/pg_upgrade/tests/9*.sql; do psql_admin -f "$f"; done + +# arm64 packages are not on archive.ubuntu.com +sed -i 's|http://[a-z]*.ubuntu.com/ubuntu|http://ports.ubuntu.com/ubuntu-ports|' /etc/apt/sources.list.d/*.sources + +IS_LOCAL_UPGRADE=true NIX_FLAKE_VERSION=local "$scripts/initiate.sh" 17 + +# complete.sh starts the new server from here, which a nix upgrade fills with only a version file +rm -rf /tmp/pg_upgrade_bin/17 +ln -s "$PG_UPGRADE_STORE_PATH" /tmp/pg_upgrade_bin/17 +# Restart=always would bring the old server back, and complete.sh moves /data_migration to /data, which must not exist +systemctl disable --now postgresql +mv /data /data.old +rm -f /tmp/pg-upgrade-status +IS_CI=true "$scripts/complete.sh" + +[ "$(psql_admin -At -c 'show server_version_num')" -ge 170000 ] +for f in tests/pg_upgrade/tests/0*.sql; do + out=$(psql_admin -At -f "$f") + echo "$out" + if grep -q '^not ok' <<<"$out"; then exit 1; fi +done +EOF