diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 810bfb8bb0..6202a1efdc 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -8,6 +8,7 @@ on: paths: - '.github/workflows/publish-nix-pgupgrade-scripts.yml' - 'ansible/vars.yml' + - 'ansible/files/admin_api_scripts/pg_upgrade_scripts/**' workflow_dispatch: inputs: postgresVersion: diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh new file mode 100755 index 0000000000..1b464436f3 --- /dev/null +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/refresh_collation.sh @@ -0,0 +1,416 @@ +#! /usr/bin/env bash + +## Rebuilds affected indexes and refreshes recorded collation versions after an +## AMI ships new glibc/ICU over existing data. Invoked on demand by adminapi (not +## a boot service); the exit code is the signal: 0 = work done, or a legitimate +## no-op (replica, pre-PG15); 1 = something failed, OR preconditions could not be +## established (Postgres unreachable) so we don't actually know the collation +## state. Per-statement errors don't abort — we fix what we can, +## then exit non-zero (like the sibling pg_upgrade scripts). Reindex runs BEFORE +## refresh so a stale index is never masked by an updated catalog; a failed reindex +## skips that database's refresh to preserve the signal. +## +## SECURITY: object names never reach the shell. Enumeration returns integer OIDs +## only; DDL is built and run server-side via format('%I',...) + \gexec, so a +## hostile collation/index name (any user with CREATE on a schema) can't inject a +## psql meta-command (\!) to get a root shell via this script's sudoers grant. + +set -uo pipefail # deliberately NOT -e: per-statement errors are handled inline + +# pg_database_collation_actual_version() + datcollversion are PG15+; nothing to +# compare against below that. (The collation-level function is older, but this +# database-level gate sets the floor.) +MIN_SERVER_VERSION_NUM=150000 + +# libc always; ICU always. Reindex-before-refresh makes ICU safe. +PROVIDERS="'c', 'i'" + +# Caps REINDEX CONCURRENTLY's brief locks so one idle-in-transaction client can't +# block it forever. Passed via PGOPTIONS, not a ;-joined SET — that would open a +# transaction block, which REINDEX CONCURRENTLY refuses to run inside. +REINDEX_LOCK_TIMEOUT_MS=2000 + +# Set by any enumeration/reindex/refresh failure; becomes the exit code. +SCRIPT_FAILED=0 + +# Advisory snapshot consumed by adminapi (api/refresh_collation.go). Rewritten +# fresh every run; readable by the adminapi user (0644 — index names, not secret). +ADVISORY_FILE="/tmp/collation-refresh-status.json" +ADVISORY_NDJSON="" # per-run temp NDJSON accumulator; set in main() +ADVISORY_DBS="" # per-run temp file of databases that produced advisories + +log() { + echo "[$(date '+%Y-%m-%d %H:%M:%S')] refresh_collation: $1" +} + +# ON_ERROR_STOP makes psql exit non-zero on any SQL error (incl. inside \gexec) — +# that is how per-statement failure is detected below. +run_sql() { + psql -h localhost -U supabase_admin -v ON_ERROR_STOP=1 --no-psqlrc "$@" +} + +# Retry to guard against a not-yet-ready Postgres socket. +retry() { + local attempts=$1 + shift + local i=0 + until "$@"; do + i=$((i + 1)) + if [ "$i" -ge "$attempts" ]; then + return 1 + fi + sleep 1 + done + return 0 +} + +# Wrap a db name in dbname='...' (escaping \ and ') so characters special to -d +# parsing (=, spaces, quotes) stay part of a literal name, not a conninfo fragment. +conninfo_for_db() { + local d="$1" + d="${d//\\/\\\\}" + d="${d//\'/\\\'}" + printf "dbname='%s'" "$d" +} + +# --- Enumeration queries: emit integer OIDs only (never object names) ---------- + +# Leaf indexes ('i') in user schemas depending on a stale collation (explicit or +# the db default). Partitioned parents ('I') have no storage; catalogs use +# version-less collations; temp schemas are skipped (reindexing another session's +# temp index fails). +# +# SCOPE / KNOWN LIMITATION (follow-up): detection keys off pg_index.indcollation — +# the collations of the index's KEY COLUMNS only. Collation dependencies that live +# elsewhere are NOT detected and NOT rebuilt: partial-index predicates +# (pg_index.indpred), CHECK constraints (pg_constraint), and partition bound +# expressions (pg_class.relpartbound). The refresh step below bumps the recorded +# version for EVERY stale collation regardless of where it is used, so those +# dependencies get stamped "refreshed" without any revalidation. The intended +# follow-up surfaces them via the adminapi advisory channel rather than silently +# refreshing. See: +# https://github.com/supabase/postgres/pull/2343#discussion_r3756738261 +# Shared WITH clause: collations (affected_coll) and the current db's default +# (affected_default) whose recorded version is stale. Used by both the reindex +# enumeration and the advisory query so their notion of "affected" cannot drift. +_affected_ctes() { + cat <>"$ADVISORY_DBS" + return + fi + + # Exclusion/invalid indexes we cannot rebuild automatically: record an advisory, + # do NOT fail, and skip this database's refresh entirely (named collations AND + # the db default in section c) so a stale version is never stamped over an + # un-rebuilt index. The advisory persists until the customer reindexes manually. + local advisories + advisories="$(run_sql -d "$conn" -Atq -c "$(affected_advisory_sql)")" + rc=$? + if [ "$rc" -ne 0 ]; then + log "WARN could not enumerate advisory indexes on $db (psql rc=$rc); skipping refresh to preserve signal" + SCRIPT_FAILED=1 + printf '%s\n' "$db" >>"$ADVISORY_DBS" + return + fi + if [ -n "$advisories" ]; then + printf '%s\n' "$advisories" >>"$ADVISORY_NDJSON" + printf '%s\n' "$db" >>"$ADVISORY_DBS" + log "advisory: $db has exclusion/invalid indexes needing manual REINDEX; skipping this DB's collation refresh" + return + fi + + oids="$(run_sql -d "$conn" -Atq -c "$(stale_collation_oids_sql)")" + rc=$? + if [ "$rc" -ne 0 ]; then + log "WARN could not enumerate stale collations on $db (psql rc=$rc)" + SCRIPT_FAILED=1 + return + fi + while IFS= read -r oid; do + [ -z "$oid" ] && continue + if ! [[ $oid =~ ^[0-9]+$ ]]; then + log "WARN ignoring non-numeric collation oid '$oid' on $db" + SCRIPT_FAILED=1 + continue + fi + log "refresh collation $db :: collation oid $oid" + if ! refresh_collation_by_oid "$conn" "$oid"; then + log "WARN collation refresh failed on $db :: collation oid $oid (continuing)" + SCRIPT_FAILED=1 + fi + done <<<"$oids" +} + +main() { + # Fresh, readable, empty snapshot up front; overwritten at the end if findings. + printf '[]' >"$ADVISORY_FILE" 2>/dev/null || true + chmod 0644 "$ADVISORY_FILE" 2>/dev/null || true + ADVISORY_NDJSON="$(mktemp)" + ADVISORY_DBS="$(mktemp)" + trap 'rm -f "$ADVISORY_NDJSON" "$ADVISORY_DBS"' EXIT + + if ! retry 8 pg_isready -h localhost -U supabase_admin -d postgres; then + log "postgres not ready after retries; could not run (reporting failure)" + SCRIPT_FAILED=1 + return + fi + + # Read replica: catalogs are read-only (changes stream from the primary) and + # every REINDEX/ALTER would fail "read-only transaction"; skip. + local in_recovery + in_recovery="$(run_sql -d postgres -Atq -c "select pg_is_in_recovery();")" || in_recovery="" + if [ "$in_recovery" = "t" ]; then + log "server is in recovery (replica); skipping" + return 0 + fi + + # Unknown/non-numeric version must not pass the floor check — skip, don't proceed. + local svn + svn="$(run_sql -d postgres -Atq -c "select current_setting('server_version_num');")" || svn="" + if ! [[ $svn =~ ^[0-9]+$ ]]; then + log "could not read a numeric server_version_num (got '${svn}'); skipping" + return 0 + fi + if [ "$svn" -lt "$MIN_SERVER_VERSION_NUM" ]; then + log "server_version_num=$svn < $MIN_SERVER_VERSION_NUM; collation version tracking unavailable, skipping" + return 0 + fi + + local dbs rc db oids oid dbname adv_json + dbs="$(run_sql -d postgres -Atq -c "select datname from pg_database where datallowconn and datname <> 'template0' order by datname;")" + rc=$? + if [ "$rc" -ne 0 ]; then + log "WARN could not enumerate databases (psql rc=$rc)" + SCRIPT_FAILED=1 + return + fi + while IFS= read -r db; do + [ -z "$db" ] && continue + process_database "$db" + done <<<"$dbs" + + # c. Database-default versions (shared catalog), after every db's default- + # collated indexes were reindexed above. Includes template0. + oids="$(run_sql -d postgres -Atq -c "$(stale_db_default_oids_sql)")" + rc=$? + if [ "$rc" -ne 0 ]; then + log "WARN could not enumerate stale database defaults (psql rc=$rc)" + SCRIPT_FAILED=1 + else + while IFS= read -r oid; do + [ -z "$oid" ] && continue + if ! [[ $oid =~ ^[0-9]+$ ]]; then + log "WARN ignoring non-numeric database oid '$oid'" + SCRIPT_FAILED=1 + continue + fi + dbname="$(run_sql -d postgres -Atq -c "select datname from pg_database where oid = $oid;")" + rc=$? + if [ "$rc" -ne 0 ] || [ -z "$dbname" ]; then + log "WARN could not resolve datname for database oid $oid (psql rc=$rc); skipping db-default refresh to avoid masking advisories" + SCRIPT_FAILED=1 + continue + fi + if grep -qxF "$dbname" "$ADVISORY_DBS"; then + log "skipping db-default refresh for $dbname: outstanding collation advisories" + continue + fi + log "refresh (db default) :: database oid $oid" + if ! refresh_db_default_by_oid "$oid"; then + log "WARN database-default refresh failed :: database oid $oid (continuing)" + SCRIPT_FAILED=1 + fi + done <<<"$oids" + fi + + # Assemble the JSON array snapshot from per-DB NDJSON. Fail-open: any problem + # leaves the "[]" written at entry. jq matches the sibling scripts' JSON tooling. + if [ -s "$ADVISORY_NDJSON" ]; then + if adv_json="$(jq -s '.' "$ADVISORY_NDJSON" 2>/dev/null)"; then + printf '%s\n' "$adv_json" >"${ADVISORY_FILE}.tmp" && + mv "${ADVISORY_FILE}.tmp" "$ADVISORY_FILE" + chmod 0644 "$ADVISORY_FILE" 2>/dev/null || true + else + log "WARN could not assemble advisory JSON; leaving prior snapshot" + SCRIPT_FAILED=1 + fi + fi + + log "done (failed=$SCRIPT_FAILED)" +} + +main +exit "$SCRIPT_FAILED" diff --git a/ansible/files/adminapi.sudoers.conf b/ansible/files/adminapi.sudoers.conf index e6d27bd104..ab10665a3c 100644 --- a/ansible/files/adminapi.sudoers.conf +++ b/ansible/files/adminapi.sudoers.conf @@ -14,6 +14,7 @@ Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl st %adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/check.sh %adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/common.sh %adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/pgsodium_getkey.sh +%adminapi ALL= NOPASSWD: /etc/adminapi/pg_upgrade_scripts/refresh_collation.sh %adminapi ALL= NOPASSWD: /usr/bin/systemctl daemon-reload # pgBackRest wrapper scripts: constrained helpers called by supabase-admin-agent. # pgdata-chown runs as root (default); pgdata-signal runs as postgres so it can diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index 6affb840c7..39d43569be 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -66,6 +66,7 @@ - { file: "prepare.sh" } - { file: "pgsodium_getkey.sh" } - { file: "common.sh" } + - { file: "refresh_collation.sh" } - name: adminapi - create service file template: diff --git a/audit-specs/baselines/baseline.yml b/audit-specs/baselines/baseline.yml index 0c7810c629..395366c1da 100644 --- a/audit-specs/baselines/baseline.yml +++ b/audit-specs/baselines/baseline.yml @@ -145,6 +145,12 @@ file: owner: "1006" group: "0" filetype: file + /etc/adminapi/pg_upgrade_scripts/refresh_collation.sh: + exists: true + mode: "0755" + owner: "1006" + group: "0" + filetype: file /etc/alternatives/README: exists: true mode: "0644"