From 9a3aa4b9d979e904b9b4cd1338109839ba39ac50 Mon Sep 17 00:00:00 2001 From: pgnickb Date: Tue, 29 Sep 2026 15:26:21 +0200 Subject: [PATCH] feat(coredump): capture PostgreSQL-only core dumps on the OrioleDB AMI Scope core capture to postgresql.service only: LimitCORE=infinity via a drop-in, and a machine-wide DefaultLimitCORE=0 so no other service is affected. Also set fs.suid_dumpable=2, without which the AppArmor-confined postmaster's exec is treated as a kernel "secure exec" and made entirely non-dumpable, so no core is produced at all regardless of LimitCORE. Gated behind a new is_psql_oriole fact in setup-postgres.yml (OrioleDB only). Add the OrioleDB-only processing pipeline: a systemd path unit (with a timer fallback) that runs process-orioledb-coredumps.sh whenever a new core appears. The script filters to postgres-owned cores, extracts a GDB backtrace/lock-state bundle (cmds.gdb, matching orioledb/ci's own debugging script) into a redacted diagnostic bundle, then deletes the raw core, retrying/quarantining on repeated failure. Also sets the kernel's default coredump_filter to 0x31 (excluding shared_buffers) via a GRUB parameter, so it applies from every process's first fork onward. This is what actually fixes the filter: a per-unit CoredumpFilter= directive on postgresql.service was tried first but gets silently reset by AppArmor's unconfined->confined exec transition. --- ansible/files/coredump/cmds.gdb | 20 + ansible/files/coredump/orioledb-coredump.path | 9 + .../files/coredump/orioledb-coredump.service | 14 + .../files/coredump/orioledb-coredump.timer | 11 + .../coredump/process-orioledb-coredumps.sh | 275 ++++++++++ .../postgresql_config/coredump-storage.conf | 7 + ansible/files/postgresql_config/coredump.conf | 2 + .../disable-coredumps-by-default.conf | 2 + ansible/playbook.yml | 11 + ansible/tasks/setup-coredump-processing.yml | 161 ++++++ ansible/tasks/setup-postgres.yml | 10 - ansible/tasks/setup-supabase-internal.yml | 19 +- ansible/tasks/setup-tuned.yml | 4 + nix/packages/postgres-env.nix | 8 +- testinfra/test_ami_nix.py | 469 ++++++++++++++++++ 15 files changed, 1006 insertions(+), 16 deletions(-) create mode 100644 ansible/files/coredump/cmds.gdb create mode 100644 ansible/files/coredump/orioledb-coredump.path create mode 100644 ansible/files/coredump/orioledb-coredump.service create mode 100644 ansible/files/coredump/orioledb-coredump.timer create mode 100644 ansible/files/coredump/process-orioledb-coredumps.sh create mode 100644 ansible/files/postgresql_config/coredump-storage.conf create mode 100644 ansible/files/postgresql_config/coredump.conf create mode 100644 ansible/files/postgresql_config/disable-coredumps-by-default.conf create mode 100644 ansible/tasks/setup-coredump-processing.yml diff --git a/ansible/files/coredump/cmds.gdb b/ansible/files/coredump/cmds.gdb new file mode 100644 index 0000000000..93b4599453 --- /dev/null +++ b/ansible/files/coredump/cmds.gdb @@ -0,0 +1,20 @@ +# Matches orioledb/ci/cmds.gdb (OrioleDB's own CI crash-debugging script). +# debug-file-directory/file/core-file are set by the caller before this +# file is sourced (see process-orioledb-coredumps.sh). +# +# GDB aborts the rest of a sourced script on the first command error, so +# order matters here: the reliable, high-value output runs first; the +# OrioleDB-internal lock-state dumps run last, since they can fail if +# orioledb.so's own debug symbols aren't available (a known, separate issue) +thread apply all bt full +up 99999 +set $i=0 +set $end=argc +while ($i < $end) +p argv[$i++] +end +info sharedlibrary +info registers +eval "p *((LWLockHandle (*) [%u]) held_lwlocks)", num_held_lwlocks +eval "p *((MyLockedPage (*) [%u]) myLockedPages)", numberOfMyLockedPages +quit diff --git a/ansible/files/coredump/orioledb-coredump.path b/ansible/files/coredump/orioledb-coredump.path new file mode 100644 index 0000000000..c9a5ce83d1 --- /dev/null +++ b/ansible/files/coredump/orioledb-coredump.path @@ -0,0 +1,9 @@ +[Unit] +Description=Watch for PostgreSQL/OrioleDB core dumps + +[Path] +PathChanged=/var/lib/systemd/coredump +Unit=orioledb-coredump.service + +[Install] +WantedBy=multi-user.target diff --git a/ansible/files/coredump/orioledb-coredump.service b/ansible/files/coredump/orioledb-coredump.service new file mode 100644 index 0000000000..49e574438d --- /dev/null +++ b/ansible/files/coredump/orioledb-coredump.service @@ -0,0 +1,14 @@ +[Unit] +Description=Process PostgreSQL/OrioleDB core dumps into a redacted diagnostic bundle +After=systemd-coredump@.service + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/process-orioledb-coredumps.sh +User=root +Group=root +NoNewPrivileges=yes +PrivateTmp=yes +ProtectSystem=strict +ProtectHome=yes +ReadWritePaths=/var/lib/systemd/coredump /var/lib/orioledb-coredumps /run diff --git a/ansible/files/coredump/orioledb-coredump.timer b/ansible/files/coredump/orioledb-coredump.timer new file mode 100644 index 0000000000..7b1a25f419 --- /dev/null +++ b/ansible/files/coredump/orioledb-coredump.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Periodic fallback sweep for PostgreSQL/OrioleDB core dumps + +[Timer] +OnBootSec=60min +RandomizedDelaySec=15min +OnUnitActiveSec=15min +Unit=orioledb-coredump.service + +[Install] +WantedBy=timers.target diff --git a/ansible/files/coredump/process-orioledb-coredumps.sh b/ansible/files/coredump/process-orioledb-coredumps.sh new file mode 100644 index 0000000000..2ce913f619 --- /dev/null +++ b/ansible/files/coredump/process-orioledb-coredumps.sh @@ -0,0 +1,275 @@ +#!/bin/bash +# Process PostgreSQL/OrioleDB core dumps captured by systemd-coredump into a +# text summary. +# +# Key ideas: +# - Only cores of the postgres binary are processed; any other core left +# untouched +# - No environment variables are ever collected. The GDB extraction (see +# cmds.gdb, matching OrioleDB's own CI debugging script) does run +# `thread apply all bt full`, which prints local variable values and can +# surface fragments of in-memory data (buffer/tuple pointers etc.) - this +# is a deliberate, reviewed trade-off in favor of debuggability, not an +# oversight. +# - Cores are deleted after a successful run or quarantined (metadata only) +# after MAX_ATTEMPTS failures. + +set -euo pipefail + +STATE_DIR=/var/lib/orioledb-coredumps/state +OUTPUT_DIR=/var/lib/orioledb-coredumps/diagnostics +QUARANTINE_DIR=/var/lib/orioledb-coredumps/quarantine + +MAX_ATTEMPTS=3 +EXTRACTION_TIMEOUT=120 +MAX_AGE_DAYS=7 +MAX_TOTAL_BYTES=$((300 * 1024 * 1024)) # independent of systemd-coredump's own MaxUse + +GDB_DEBUG_DIR=/var/lib/postgresql/.nix-profile/lib/debug +GDB_CMDS_FILE=/usr/local/sbin/orioledb-coredump-cmds.gdb +PGDATA_CURRENT_LOGFILES=/var/lib/postgresql/data/current_logfiles + +log() { + echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] $*" +} + +# Only ever run under orioledb-coredump.service, so this always executes with +# the unit's hardening (privs, sandboxing) rather than whatever an interactive +# shell happens to have. +if [[ -z ${INVOCATION_ID:-} ]]; then + echo "This script must be run through orioledb-coredump.service" >&2 + exit 1 +fi + +# orioledb.so has no fixed, predictable path either - there is no +# /usr/lib/postgresql/lib mirror of it. It lives in the same nix store +# derivation as the resolved postgres executable, just under lib/ instead +# of bin/, e.g. .../postgresql-and-plugins-17_20/{bin/.postgres-wrapped, +# lib/orioledb.so} - so derive it from $exe rather than guessing a path. +orioledb_lib_path() { + local exe_dir + exe_dir=$(dirname "$(dirname "$1")") + printf '%s/lib/orioledb.so' "$exe_dir" +} + +# coredumpctl on this systemd version (255.4) has no "rm"/"delete" verb - the +# only reliable way to remove a core is to delete its on-disk Filename path +# directly. Returns success only if the path is actually gone afterward. +delete_core() { + local path="$1" + [ -z "$path" ] && return 1 + rm -f -- "$path" + [ ! -e "$path" ] +} + +# The active postgresql log file has an unpredictable name and can be +# csvlog, stderr-text, or both depending on config (this AMI defaults to +# csvlog-only, e.g. /var/log/postgresql/postgresql.csv - the stderr-format +# postgresql.log stops receiving anything the moment the logging collector +# switches over at startup). PGDATA/current_logfiles is postgres's own, +# always-current record of the real path(s); prefer csvlog, fall back to +# stderr. Either format still starts each line with a literal timestamp, so +# the grep -F substring match below works unchanged either way. +current_postgres_log() { + [ -f "$PGDATA_CURRENT_LOGFILES" ] || return 0 + awk '$1 == "csvlog" {p = $2} $1 == "stderr" && !p {p = $2} END {print p}' "$PGDATA_CURRENT_LOGFILES" +} + +enforce_retention() { + find "$OUTPUT_DIR" -maxdepth 1 -type f -mtime "+${MAX_AGE_DAYS}" -delete 2>/dev/null || true + + while true; do + total=$(du -sb "$OUTPUT_DIR" 2>/dev/null | cut -f1) + [ -z "$total" ] && break + [ "$total" -le "$MAX_TOTAL_BYTES" ] && break + oldest=$(find "$OUTPUT_DIR" -maxdepth 1 -type f -printf '%T@ %p\n' 2>/dev/null | sort -n | head -1 | cut -d' ' -f2-) + [ -z "$oldest" ] && break + log "retention: removing oldest bundle $oldest to stay under ${MAX_TOTAL_BYTES} bytes" + rm -f "$oldest" + done +} + +# Handles one candidate crash, given its PID: decide whether it's ours to +# process, extract a diagnostic bundle via GDB, then delete the raw core. +# (1) look up the crash via coredumpctl +# (2) decide keep/ignore/quarantine based on prior attempts +# (3) export the core and run GDB against it +# (4) write the bundle and delete the raw core. Returns 1 only for failures +# worth retrying next run (main() logs those); every other outcome is +# ignored, quarantined, or successfully processed - returns 0. +process_one() { + local pid="$1" + + # `coredumpctl --json=short info` silently IGNORES --json on this + # systemd version (255.4) and falls back to the old plain-text format - + # confirmed against a real build, not just docs. jq then fails to parse + # it as JSON at all, so every field below would come out empty (wrong + # boot_id, no storage path, core never gets deleted) with no visible + # error anywhere except this script's own log. `coredumpctl --json=short + # list` (used in main()) is unaffected - only the info verb's --json + # support is broken on this version. Query the journal directly instead + # - COREDUMP_* fields are systemd-coredump's own structured record of + # the crash, independent of coredumpctl's info-rendering bug, and + # journalctl's -o json has no version-specific gap like this. Only the + # fields actually used below are requested - COREDUMP_ENVIRON in + # particular is deliberately left out, matching "no environment + # variables are ever collected" above. + local journal_json + journal_json=$(journalctl -o json \ + MESSAGE_ID=fc2e22bc6ee647b6b90729ab34a250b1 \ + "COREDUMP_PID=${pid}" \ + --output-fields=COREDUMP_EXE,COREDUMP_SIGNAL,COREDUMP_SIGNAL_NAME,COREDUMP_TIMESTAMP,COREDUMP_FILENAME,_BOOT_ID \ + 2>/dev/null | tail -n 1) + if [ -z "$journal_json" ]; then + log "pid ${pid}: no matching journal entry for this coredump" + return 1 + fi + + # join() (not @tsv) because @tsv double-escapes backslashes, which would + # corrupt systemd-coredump filenames (e.g. "\x2e") and prevent core + # deletion - none of these fields can contain a literal tab/newline. + local exe boot_id signal signal_name timestamp_usec filename + IFS=$'\t' read -r exe boot_id signal signal_name timestamp_usec filename < <( + jq -r '[.COREDUMP_EXE, ._BOOT_ID, .COREDUMP_SIGNAL, .COREDUMP_SIGNAL_NAME, .COREDUMP_TIMESTAMP, (.COREDUMP_FILENAME // "")] | join("\t")' <<<"$journal_json" + ) + # Unlike coredumpctl info's derived "Storage: present/missing" status, + # COREDUMP_FILENAME is just the path recorded at capture time, with no + # guarantee it's still on disk - delete_core already treats "already + # gone" as success (rm -f + existence check), so no separate presence + # check is needed here. + local storage_path="$filename" + # PID alone isn't a safe dedup key long-term (PIDs get reused across + # boots), so pair it with boot ID - matches "state keyed by boot ID plus + # dump identifier" from the original design. + local key="${boot_id}-${pid}" + local state_file="${STATE_DIR}/${key}" + + # .done means "final decision made, never look at this dump again" + # (processed successfully or quarantined - non-postgres cores are never + # enumerated in the first place, see main()'s list filter). + # .attempts only counts *failed* tries, to cap retries before quarantine. + [ -f "${state_file}.done" ] && return 0 + + local attempts=0 + [ -f "${state_file}.attempts" ] && attempts=$(cat "${state_file}.attempts") + if [ "$attempts" -ge "$MAX_ATTEMPTS" ]; then + log "pid ${pid}: exceeded ${MAX_ATTEMPTS} attempts, discarding core (metadata kept in ${QUARANTINE_DIR})" + jq . <<<"$journal_json" >"${QUARANTINE_DIR}/${key}.info" + if delete_core "$storage_path"; then + log "pid ${pid}: raw core deleted" + else + log "pid ${pid}: WARNING - could not delete raw core at '${storage_path}'" + fi + touch "${state_file}.done" + return 0 + fi + + # From here on we're committed to actually processing this dump, so + # count it as an attempt before doing any of the risky (slow, can fail) + # work below - a crash/timeout past this point still gets retried, up + # to MAX_ATTEMPTS + echo $((attempts + 1)) >"${state_file}.attempts" + + # coredumpctl stores the core compressed; pull a private, working copy + # out into a root-only scratch dir before handing it to GDB. The trap + # guarantees that scratch dir is removed when this function returns, no + # matter which of the several `return`s below fires. `trap ... RETURN` + # is NOT scoped to this function - it fires on the next return from ANY + # function (confirmed: without the self-clearing `trap - RETURN` here, + # main()'s own `return "$rc"` re-fires this same trap, referencing a + # $tmpdir that's gone out of scope, which is an unbound-variable crash + # under `set -u` - so it must clear itself once it's run. + local tmpdir + tmpdir=$(mktemp -d /tmp/coredump-XXXXXX) + chmod 700 "$tmpdir" + trap 'rm -rf "$tmpdir"; trap - RETURN' RETURN + + if ! timeout "$EXTRACTION_TIMEOUT" coredumpctl dump "$pid" --output "${tmpdir}/core" >/dev/null 2>&1; then + log "pid ${pid}: export failed or timed out" + return 1 + fi + + # Everything from here to the closing "}" is the diagnostic bundle + # itself, one section at a time, redirected straight to $bundle - + # there's no in-memory buffering of it, so a slow/hanging step just + # shows up as a truncated file rather than blocking the whole write. + # Timestamp from coredumpctl JSON is microseconds since the epoch, so it + # converts straight to a UTC date/time with no string parsing - and that + # same "YYYY-MM-DD HH:MM:SS" form is what postgres's own log line prefix + # starts with, so it doubles as the grep key below. + local crash_timestamp + crash_timestamp=$(date -u -d "@$((timestamp_usec / 1000000))" '+%Y-%m-%d %H:%M:%S') + + local bundle="${OUTPUT_DIR}/${key}.txt" + { + echo "== OrioleDB/PostgreSQL coredump diagnostic bundle ==" + echo "generated: $(date -u '+%Y-%m-%dT%H:%M:%SZ')" + echo "pid: ${pid}" + echo "boot_id: ${boot_id}" + echo "signal: ${signal} (${signal_name})" + echo "crash_timestamp: ${crash_timestamp} UTC" + echo "executable: ${exe}" + echo + + echo "== build ids ==" + echo "postgres (${exe}):" + readelf -n "$exe" 2>/dev/null | grep 'Build ID' || echo " (could not read build id)" + orioledb_lib=$(orioledb_lib_path "$exe") + if [ -f "$orioledb_lib" ]; then + echo "orioledb.so (${orioledb_lib}):" + readelf -n "$orioledb_lib" 2>/dev/null | grep 'Build ID' || echo " (could not read build id)" + fi + echo + + echo "== gdb backtrace (full), lwlocks, locked pages, argv, shared libraries, registers ==" + timeout "$EXTRACTION_TIMEOUT" gdb --batch -quiet \ + -ex "set debug-file-directory ${GDB_DEBUG_DIR}" \ + -ex "file ${exe}" \ + -ex "core-file ${tmpdir}/core" \ + -x "$GDB_CMDS_FILE" \ + 2>&1 || echo "(gdb extraction failed or timed out)" + echo + + echo "== postgresql.log excerpt around crash ==" + postgres_log=$(current_postgres_log) + if [ -n "$postgres_log" ] && [ -f "$postgres_log" ]; then + grep -F "$crash_timestamp" -A 5 -B 20 "$postgres_log" 2>/dev/null | tail -200 || + echo "(no log lines found matching ${crash_timestamp} in ${postgres_log})" + else + echo "(no crash timestamp or log file available)" + fi + } >"$bundle" + chmod 600 "$bundle" + + # Bundle is written either way at this point, even if the raw-core + # delete below fails - we don't want a delete failure to make us + # reprocess an already-complete bundle next run. + touch "${state_file}.done" + if delete_core "$storage_path"; then + log "pid ${pid}: wrote ${bundle}, deleted raw core" + else + log "pid ${pid}: wrote ${bundle}, but WARNING - could not delete raw core at '${storage_path}'" + fi +} + +main() { + mkdir -p "$STATE_DIR" "$OUTPUT_DIR" "$QUARANTINE_DIR" + chmod 700 "$STATE_DIR" "$OUTPUT_DIR" "$QUARANTINE_DIR" + + enforce_retention + + # Enumerate via coredumpctl and filter out non-postgres cores + local rc=0 + local pid + while read -r pid; do + [ -z "$pid" ] && continue + process_one "$pid" || { + log "pid ${pid}: processing failed, will retry on next run" + rc=1 + } + done < <(coredumpctl --no-pager --json=short list 2>/dev/null | jq -r '.[] | select(.corefile == "present" and (.exe | endswith("/.postgres-wrapped"))) | .pid') + + return "$rc" +} + +main diff --git a/ansible/files/postgresql_config/coredump-storage.conf b/ansible/files/postgresql_config/coredump-storage.conf new file mode 100644 index 0000000000..a715ad7c11 --- /dev/null +++ b/ansible/files/postgresql_config/coredump-storage.conf @@ -0,0 +1,7 @@ +[Coredump] +Storage=external +Compress=yes +ProcessSizeMax=256M +ExternalSizeMax=256M +MaxUse=512M +KeepFree=5G diff --git a/ansible/files/postgresql_config/coredump.conf b/ansible/files/postgresql_config/coredump.conf new file mode 100644 index 0000000000..7c45cd49e5 --- /dev/null +++ b/ansible/files/postgresql_config/coredump.conf @@ -0,0 +1,2 @@ +[Service] +LimitCORE=infinity diff --git a/ansible/files/postgresql_config/disable-coredumps-by-default.conf b/ansible/files/postgresql_config/disable-coredumps-by-default.conf new file mode 100644 index 0000000000..9f1d52b50a --- /dev/null +++ b/ansible/files/postgresql_config/disable-coredumps-by-default.conf @@ -0,0 +1,2 @@ +[Manager] +DefaultLimitCORE=0 diff --git a/ansible/playbook.yml b/ansible/playbook.yml index 93bc178060..2c215ba872 100644 --- a/ansible/playbook.yml +++ b/ansible/playbook.yml @@ -30,6 +30,13 @@ - set_fact: parallel_jobs: 16 + # is_psql_oriole is also (re)computed in stage2-setup-postgres.yml, but the + # coredump-processing import below runs in both stage1 and stage2, so the + # fact needs to exist before stage2-only tasks have had a chance to set it. + - name: Determine postgres flavor from psql_version + ansible.builtin.set_fact: + is_psql_oriole: "{{ psql_version is defined and psql_version == 'psql_orioledb-17' }}" + - name: Install tuned import_tasks: tasks/setup-tuned.yml @@ -39,6 +46,10 @@ - name: Install Postgres from source import_tasks: tasks/setup-postgres.yml + - name: Install PostgreSQL/OrioleDB coredump capture and processing + when: is_psql_oriole + import_tasks: tasks/setup-coredump-processing.yml + - name: Install PgBouncer import_tasks: tasks/setup-pgbouncer.yml tags: diff --git a/ansible/tasks/setup-coredump-processing.yml b/ansible/tasks/setup-coredump-processing.yml new file mode 100644 index 0000000000..6f2c30e5a1 --- /dev/null +++ b/ansible/tasks/setup-coredump-processing.yml @@ -0,0 +1,161 @@ +--- +# Capture: lets PostgreSQL actually produce a core on crash. Depends on +# setup-postgres.yml having already created +# /etc/systemd/system/postgresql.service.d (used by the drop-in below). +# Runs in both stage1 and stage2 - is_psql_oriole is already known by then +# (see playbook.yml's pre-tasks). +- name: Set up PostgreSQL coredump capture (OrioleDB only) + when: is_psql_oriole + block: + # Pin the systemd-coredump group/user before installing the package: its + # postinst creates them with a dynamically-assigned system gid/uid (whatever + # is next free at that point), which is not reproducible across AMI variants + # that install a different set of packages before this point - it has + # collided with the vector group's own pinned gid (989) on some builds. + - name: Add systemd-coredump system group + ansible.builtin.group: + name: systemd-coredump + gid: 986 + system: yes + + - name: Add systemd-coredump system user + ansible.builtin.user: + name: systemd-coredump + uid: 986 + group: systemd-coredump + system: yes + create_home: false + shell: /usr/sbin/nologin + + - name: Ensure systemd-coredump is installed + ansible.builtin.apt: + name: systemd-coredump + + - name: Copy PostgreSQL coredump systemd drop-in + ansible.builtin.copy: + dest: '/etc/systemd/system/postgresql.service.d/coredump.conf' + group: 'root' + mode: '0644' + owner: 'root' + src: 'files/postgresql_config/coredump.conf' + + - name: Create a systemd system.conf.d dir for the machine-wide coredump default + ansible.builtin.file: + group: 'root' + mode: '0755' + owner: 'root' + path: '/etc/systemd/system.conf.d' + state: 'directory' + + - name: Copy machine-wide default core limit (postgres is the only exception) + ansible.builtin.copy: + dest: '/etc/systemd/system.conf.d/disable-coredumps-by-default.conf' + group: 'root' + mode: '0644' + owner: 'root' + src: 'files/postgresql_config/disable-coredumps-by-default.conf' + + - name: Create a systemd-coredump conf.d dir for PostgreSQL storage limits + ansible.builtin.file: + group: 'root' + mode: '0755' + owner: 'root' + path: '/etc/systemd/coredump.conf.d' + state: 'directory' + + - name: Copy systemd-coredump storage limits + ansible.builtin.copy: + dest: '/etc/systemd/coredump.conf.d/postgres.conf' + group: 'root' + mode: '0644' + owner: 'root' + src: 'files/postgresql_config/coredump-storage.conf' + +# Processing: turns a captured core into a diagnostic bundle. stage2-only - +# unlike capture above, this needs the debug symbols that +# stage2-setup-postgres.yml installs, so there's no point enabling it earlier. +- name: Set up PostgreSQL/OrioleDB coredump processing (OrioleDB only) + when: stage2 and is_psql_oriole + block: + # Sets the kernel's *default* coredump_filter (normally 0x33) to 0x31, + # excluding shared_buffers. + - name: Set kernel default coredump_filter via GRUB + ansible.builtin.copy: + content: | + GRUB_CMDLINE_LINUX_DEFAULT="${GRUB_CMDLINE_LINUX_DEFAULT} coredump_filter=0x31" + dest: '/etc/default/grub.d/60-coredump-filter.cfg' + group: 'root' + mode: '0644' + owner: 'root' + notify: + - Regenerate grub configuration + + # gdb and binutils (readelf) are only used by the processor script below. + # jq parses coredumpctl's --json output instead of scraping its text + # tables. + - name: Ensure gdb, binutils, and jq are installed + ansible.builtin.apt: + name: + - gdb + - binutils + - jq + + - name: Create orioledb-coredumps state/output directories + ansible.builtin.file: + group: 'root' + mode: '0700' + owner: 'root' + path: "{{ coredump_processing_item }}" + state: 'directory' + loop: + - '/var/lib/orioledb-coredumps' + - '/var/lib/orioledb-coredumps/state' + - '/var/lib/orioledb-coredumps/diagnostics' + - '/var/lib/orioledb-coredumps/quarantine' + loop_control: + loop_var: 'coredump_processing_item' + + - name: Copy PostgreSQL/OrioleDB coredump processor script + ansible.builtin.copy: + dest: '/usr/local/sbin/process-orioledb-coredumps.sh' + group: 'root' + mode: '0700' + owner: 'root' + src: 'coredump/process-orioledb-coredumps.sh' + + - name: Copy GDB extraction commands (matches orioledb/ci/cmds.gdb) + ansible.builtin.copy: + dest: '/usr/local/sbin/orioledb-coredump-cmds.gdb' + group: 'root' + mode: '0600' + owner: 'root' + src: 'coredump/cmds.gdb' + + - name: Copy PostgreSQL/OrioleDB coredump systemd units + ansible.builtin.copy: + dest: "/etc/systemd/system/{{ coredump_unit_item }}" + group: 'root' + mode: '0644' + owner: 'root' + src: "coredump/{{ coredump_unit_item }}" + loop: + - 'orioledb-coredump.path' + - 'orioledb-coredump.timer' + - 'orioledb-coredump.service' + loop_control: + loop_var: 'coredump_unit_item' + + - name: Reload systemd for coredump processing units + ansible.builtin.systemd_service: + daemon_reload: true + + - name: Enable coredump processing path and timer units + ansible.builtin.systemd_service: + name: "{{ coredump_enable_item }}" + enabled: true + state: 'started' + loop: + - 'orioledb-coredump.path' + - 'orioledb-coredump.timer' + loop_control: + loop_var: 'coredump_enable_item' diff --git a/ansible/tasks/setup-postgres.yml b/ansible/tasks/setup-postgres.yml index 1bcc0420e4..9dd0ddcd9c 100644 --- a/ansible/tasks/setup-postgres.yml +++ b/ansible/tasks/setup-postgres.yml @@ -27,7 +27,6 @@ shell: '/bin/bash' state: 'present' system: true - become: yes - name: Create relevant directories ansible.builtin.file: @@ -153,7 +152,6 @@ ansible.builtin.apt: name: 'locales' state: 'present' - become: true - name: configure locales ansible.builtin.lineinfile: @@ -161,7 +159,6 @@ line: "{{ locale_item }}.UTF-8 UTF-8" path: '/etc/locale.gen' state: 'present' - become: true loop: - 'C' - 'en_US' @@ -171,12 +168,10 @@ - name: locale-gen ansible.builtin.command: cmd: 'locale-gen' - become: true - name: update-locale ansible.builtin.command: cmd: 'update-locale' - become: true - name: Install Postgres extensions ansible.builtin.import_tasks: @@ -259,7 +254,6 @@ loop_var: 'systemd_svc_item' - name: initialize pg required state - become: true ansible.builtin.file: group: 'postgres' owner: 'postgres' @@ -290,10 +284,8 @@ owner: 'root' path: '/etc/systemd/system/postgresql.service.d' state: 'directory' - become: true - name: Ensure PostgreSQL starts after tuned - become: true community.general.ini_file: create: true group: 'root' @@ -307,7 +299,6 @@ value: 'tuned.service' - name: Ensure PostgreSQL wants tuned - become: true community.general.ini_file: create: true group: 'root' @@ -333,7 +324,6 @@ create: true dest: '/var/lib/postgresql/.bashrc' line: "{{ lang_item }}" - become: true loop: - 'export LOCALE_ARCHIVE=/usr/lib/locale/locale-archive' - 'export LANG="en_US.UTF-8"' diff --git a/ansible/tasks/setup-supabase-internal.yml b/ansible/tasks/setup-supabase-internal.yml index 66c4ff1975..871cbf6fc4 100644 --- a/ansible/tasks/setup-supabase-internal.yml +++ b/ansible/tasks/setup-supabase-internal.yml @@ -14,11 +14,12 @@ register: vector_download until: vector_download is success -- name: install Vector for logging - apt: - deb: /tmp/vector.deb - become: true - +# Pin the vector group/user before installing the package: its postinst +# creates them with a dynamically-assigned system gid/uid (whatever is next +# free at that point), which is not reproducible across AMI variants that +# install a different set of packages before this point - it has collided +# with this pinned gid (989) after the systemd-coredump package install +# (setup-postgres.yml) shifted the "next free system gid" ahead of it. - name: add vector system group ansible.builtin.group: name: vector @@ -32,6 +33,14 @@ group: vector groups: postgres append: yes + create_home: false + home: /var/lib/vector + system: yes + +- name: install Vector for logging + apt: + deb: /tmp/vector.deb + become: true - name: create service files for Vector ansible.builtin.template: diff --git a/ansible/tasks/setup-tuned.yml b/ansible/tasks/setup-tuned.yml index 1d4fb16ce7..c27614f55d 100644 --- a/ansible/tasks/setup-tuned.yml +++ b/ansible/tasks/setup-tuned.yml @@ -107,6 +107,10 @@ value: '1048576' - option: 'fs.file-max' value: '312139770' + - option: 'fs.suid_dumpable' + # PostgreSQL's AppArmor profile triggers a "secure" exec; setting this + # to 2 ensures it can still reliably generate core dumps. + value: '2' - option: 'kernel.panic_on_oops' value: '1' - option: 'kernel.sched_autogroup_enabled' diff --git a/nix/packages/postgres-env.nix b/nix/packages/postgres-env.nix index 7c4153f2b2..0ae009fff7 100644 --- a/nix/packages/postgres-env.nix +++ b/nix/packages/postgres-env.nix @@ -20,7 +20,13 @@ self'.packages."postgresql_${version}_src" ] ++ lib.optionals pkgs.stdenv.isLinux [ self'.packages."postgresql_${version}_debug" ] - ++ lib.optionals (pkgs.stdenv.isLinux && version != "15") [ self'.packages.gatekeeper ]; + ++ lib.optionals (pkgs.stdenv.isLinux && version != "15") [ self'.packages.gatekeeper ] + # orioledb.so ships as a separate extension package (nix/ext/orioledb.nix), not + # part of the postgresql derivation itself, so its debug output isn't covered by + # postgresql_${version}_debug above and has to be pulled in explicitly. + ++ lib.optionals (pkgs.stdenv.isLinux && version == "orioledb-17") [ + self'.legacyPackages."psql_${version}".exts.orioledb.debug + ]; }; in { diff --git a/testinfra/test_ami_nix.py b/testinfra/test_ami_nix.py index cfd77554d6..618c521e8a 100644 --- a/testinfra/test_ami_nix.py +++ b/testinfra/test_ami_nix.py @@ -1407,3 +1407,472 @@ def test_apparmor_denies_access_to_sensitive_paths(host): f"to have succeeded.\nstdout: {result['stdout']}\nstderr: {result['stderr']}" ) print(f"Confirmed: access to {test_file} denied by AppArmor") + + +def _skip_if_not_orioledb(host): + """Skip the test if coredump capture is not installed on this AMI.""" + unit_check = run_ssh_command( + host["ssh"], "systemctl list-unit-files orioledb-coredump.path --no-legend" + ) + if "orioledb-coredump.path" not in unit_check["stdout"]: + pytest.skip( + "coredump capture not installed on this AMI (not an OrioleDB build)" + ) + + +def _skip_if_orioledb(host): + """Skip the test if coredump capture is installed on this AMI.""" + unit_check = run_ssh_command( + host["ssh"], "systemctl list-unit-files orioledb-coredump.path --no-legend" + ) + if "orioledb-coredump.path" in unit_check["stdout"]: + pytest.skip( + "coredump capture installed on this AMI (this is an OrioleDB build)" + ) + + +def test_postgresql_service_allows_unlimited_core_dumps(host): + """Verify the postgresql.service coredump drop-in sets LimitCORE=infinity.""" + _skip_if_not_orioledb(host) + result = run_ssh_command(host["ssh"], "systemctl show postgresql -p LimitCORE") + assert result["succeeded"], f"systemctl show failed: {result['stderr']}" + assert "LimitCORE=infinity" in result["stdout"], ( + f"Expected postgresql.service to have LimitCORE=infinity, got:\n{result['stdout']}" + ) + + +def test_coredump_storage_limits_configured(host): + """Verify /etc/systemd/coredump.conf.d/postgres.conf sets conservative, + bounded storage limits for the systemd-coredump storage that backs + Postgres core capture.""" + _skip_if_not_orioledb(host) + result = run_ssh_command( + host["ssh"], "cat /etc/systemd/coredump.conf.d/postgres.conf" + ) + assert result["succeeded"], ( + f"Could not read coredump storage config: {result['stderr']}" + ) + for expected in [ + "Storage=external", + "Compress=yes", + "ProcessSizeMax=", + "ExternalSizeMax=", + "MaxUse=", + "KeepFree=", + ]: + assert expected in result["stdout"], ( + f"Expected '{expected}' in /etc/systemd/coredump.conf.d/postgres.conf, " + f"got:\n{result['stdout']}" + ) + + +def test_postgres_coredump_filter_excludes_shared_buffers(host): + """Verify the running postmaster's /proc/[pid]/coredump_filter is 0x31 + (49): private mappings + ELF headers, but not anonymous-shared mappings + (shared_buffers).""" + _skip_if_not_orioledb(host) + pid = run_ssh_command(host["ssh"], "systemctl show postgresql -p MainPID --value")[ + "stdout" + ].strip() + assert pid.isdigit() and pid != "0", ( + f"Could not resolve postgresql.service MainPID: {pid}" + ) + + result = run_ssh_command(host["ssh"], f"cat /proc/{pid}/coredump_filter") + assert result["succeeded"], ( + f"Could not read coredump_filter for pid {pid}: {result['stderr']}" + ) + # /proc/[pid]/coredump_filter reads back as zero-padded hex (e.g. + # '00000031'), not the bare '31' written to it. + assert int(result["stdout"].strip(), 16) == 0x31, ( + f"Expected /proc/{pid}/coredump_filter to be '31' (0x31 = 49 decimal), " + f"got '{result['stdout'].strip()}'" + ) + + +def test_default_core_limit_is_disabled_machine_wide(host): + """Verify DefaultLimitCORE=0 is configured machine-wide.""" + _skip_if_not_orioledb(host) + result = run_ssh_command(host["ssh"], "systemctl show -p DefaultLimitCORE") + assert result["succeeded"], f"systemctl show failed: {result['stderr']}" + assert "DefaultLimitCORE=0" in result["stdout"], ( + f"Expected DefaultLimitCORE=0 machine-wide, got:\n{result['stdout']}" + ) + + +def test_coredump_storage_directory_root_only(host): + """Verify /var/lib/systemd/coredump is owned by root and not + group/other-writable.""" + _skip_if_not_orioledb(host) + result = run_ssh_command( + host["ssh"], "stat -c '%a %U:%G' /var/lib/systemd/coredump" + ) + assert result["succeeded"], f"stat failed: {result['stderr']}" + mode, owner = result["stdout"].strip().split() + assert owner.startswith("root:"), ( + f"Expected /var/lib/systemd/coredump to be owned by root, got {owner}" + ) + # Directory is world-readable (systemd's own default, 0755) - only the + # write bits matter, since core files themselves get restrictive perms. + group_other_bits = mode[-2:] + assert all(int(bit) & 0o2 == 0 for bit in group_other_bits), ( + f"Expected /var/lib/systemd/coredump to not be group/other-writable, " + f"got mode {mode}" + ) + + +def test_postgres_prestart_does_not_reset_core_limit(host): + """Verify postgres_prestart.sh never calls 'ulimit', which would risk + silently overriding the postgresql.service LimitCORE=infinity drop-in.""" + result = run_ssh_command(host["ssh"], "cat /usr/local/bin/postgres_prestart.sh") + assert result["succeeded"], f"Could not read prestart script: {result['stderr']}" + assert "ulimit" not in result["stdout"], ( + "postgres_prestart.sh must not use 'ulimit' - doing so risks silently " + "defeating the postgresql.service coredump drop-in" + ) + + +def _crash_a_backend_and_wait_for_recovery(host): + """Grab a real Postgres backend pid, SIGSEGV it, and wait for PostgreSQL's + normal crash-recovery to bring the instance back on its own. Returns the + crashed backend's pid.""" + # A backend from a one-shot query exits (and its pid becomes stale) as + # soon as the client gets its answer, so use a backend that's still busy. + run_ssh_command( + host["ssh"], + "sudo -u postgres psql -U supabase_admin -h localhost -d postgres " + "-c 'select pg_sleep(30);' >/dev/null 2>&1 &", + ) + sleep(1) + backend_pid = run_ssh_command( + host["ssh"], + "sudo -u postgres psql -U supabase_admin -h localhost -d postgres " + "-tAc \"select pid from pg_stat_activity where query = 'select pg_sleep(30);' " + "and state = 'active' limit 1;\"", + )["stdout"].strip() + assert backend_pid.isdigit(), ( + f"Could not resolve a Postgres backend pid: {backend_pid}" + ) + run_ssh_command(host["ssh"], f"sudo kill -SEGV {backend_pid}") + + recovered = False + for _ in range(30): + sleep(2) + probe = run_ssh_command( + host["ssh"], + "sudo -u postgres psql -U supabase_admin -h localhost -d postgres " + "-tAc 'select 1'", + ) + if probe["succeeded"] and probe["stdout"].strip() == "1": + recovered = True + break + assert recovered, ( + "PostgreSQL did not come back up within 60s after the induced backend crash" + ) + return backend_pid + + +def test_postgres_backend_crash_produces_core_but_unrelated_process_does_not(host): + """Verify a segfaulted Postgres backend produces a coredumpctl-visible + core, while an unrelated process crashing the same way does not.""" + _skip_if_not_orioledb(host) + before = run_ssh_command( + host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true" + ) + before_lines = set(before["stdout"].splitlines()) + + run_ssh_command( + host["ssh"], + "sudo systemd-run --unit=testinfra-unrelated-crash --collect /bin/sleep 60", + ) + sleep(1) + unrelated_pid = run_ssh_command( + host["ssh"], "systemctl show testinfra-unrelated-crash -p MainPID --value" + )["stdout"].strip() + assert unrelated_pid.isdigit() and unrelated_pid != "0", ( + f"Could not resolve the disposable unrelated unit's pid: {unrelated_pid}" + ) + run_ssh_command(host["ssh"], f"sudo kill -SEGV {unrelated_pid}") + sleep(2) + + backend_pid = _crash_a_backend_and_wait_for_recovery(host) + + after = run_ssh_command( + host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true" + ) + new_lines = set(after["stdout"].splitlines()) - before_lines + assert any("postgres" in line for line in new_lines), ( + f"Expected a new postgres core dump after SIGSEGV to backend {backend_pid}, " + f"but coredumpctl list shows:\n{after['stdout']}" + ) + + # coredumpctl logs a crash entry for any SIGSEGV regardless of whether a + # core was captured; COREFILE distinguishes "captured" from "missing". + unrelated_corefile = None + for line in new_lines: + fields = line.split() + if len(fields) >= 9 and fields[4] == unrelated_pid: + unrelated_corefile = fields[8] + break + assert unrelated_corefile in (None, "missing"), ( + f"Unrelated process {unrelated_pid} should not have produced a captured " + f"core dump (COREFILE={unrelated_corefile}):\n{after['stdout']}" + ) + + +def test_vanilla_postgres_crash_does_not_capture_core(host): + """Verify that on a vanilla (non-OrioleDB) AMI, neither a segfaulted + Postgres backend nor an unrelated process produces a captured core.""" + _skip_if_orioledb(host) + before = run_ssh_command( + host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true" + ) + before_lines = set(before["stdout"].splitlines()) + + run_ssh_command( + host["ssh"], + "sudo systemd-run --unit=testinfra-unrelated-crash --collect /bin/sleep 60", + ) + sleep(1) + unrelated_pid = run_ssh_command( + host["ssh"], "systemctl show testinfra-unrelated-crash -p MainPID --value" + )["stdout"].strip() + assert unrelated_pid.isdigit() and unrelated_pid != "0", ( + f"Could not resolve the disposable unrelated unit's pid: {unrelated_pid}" + ) + run_ssh_command(host["ssh"], f"sudo kill -SEGV {unrelated_pid}") + sleep(2) + + backend_pid = _crash_a_backend_and_wait_for_recovery(host) + + after = run_ssh_command( + host["ssh"], "sudo coredumpctl list --no-legend 2>/dev/null || true" + ) + new_lines = set(after["stdout"].splitlines()) - before_lines + + crashed = [(backend_pid, "postgres backend"), (unrelated_pid, "unrelated process")] + for pid, label in crashed: + corefile = None + for line in new_lines: + fields = line.split() + if len(fields) >= 9 and fields[4] == pid: + corefile = fields[8] + break + assert corefile in (None, "missing"), ( + f"On a vanilla (non-OrioleDB) AMI, the {label} (pid {pid}) should not " + f"have produced a captured core dump (COREFILE={corefile}):\n" + f"{after['stdout']}" + ) + + +def _resolve_postgres_binary(host): + """Return the real postgres ELF path via the live postmaster's + /proc//exe (the installed /usr/lib/postgresql/bin/postgres is a Nix + wrapper script, not the ELF itself).""" + pid = run_ssh_command(host["ssh"], "systemctl show postgresql -p MainPID --value")[ + "stdout" + ].strip() + return run_ssh_command(host["ssh"], f"sudo readlink -f /proc/{pid}/exe")[ + "stdout" + ].strip() + + +def _resolve_orioledb_lib(host, postgres_binary): + """Return orioledb.so's path, derived from the resolved postgres binary's + nix store derivation (mirrors orioledb_lib_path() in + process-orioledb-coredumps.sh).""" + exe_dir = run_ssh_command( + host["ssh"], f"dirname \"$(dirname '{postgres_binary}')\"" + )["stdout"].strip() + return f"{exe_dir}/lib/orioledb.so" + + +def _build_id_of(host, path): + """Return (build_id, readelf_output) for the binary/library at path. + build_id is None if readelf failed or found no build-id note.""" + import re + + result = run_ssh_command(host["ssh"], f"readelf -n {path}") + output = result["stdout"] + result["stderr"] + if not result["succeeded"]: + return None, output + match = re.search(r"Build ID:\s*([0-9a-f]+)", output) + return (match.group(1) if match else None), output + + +def _debug_file_exists_for_build_id(host, build_id): + """Check whether the postgres nix-profile's debug output has a + .build-id/xx/yyyy...debug file matching the given build-id.""" + prefix, rest = build_id[:2], build_id[2:] + debug_path = ( + f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug" + ) + result = run_ssh_command(host["ssh"], f"test -f {debug_path} && echo present") + return result["succeeded"] and "present" in result["stdout"] + + +def test_postgres_binary_build_id_matches_shipped_debug_symbols(host): + """Verify the installed 'postgres' binary's build-id has a matching + .build-id/xx/yyyy.debug file in the postgres-env debug output.""" + _skip_if_not_orioledb(host) + postgres_binary = _resolve_postgres_binary(host) + build_id, readelf_output = _build_id_of(host, postgres_binary) + assert build_id, ( + f"Could not read a build-id from {postgres_binary}, readelf -n output:\n" + f"{readelf_output}" + ) + assert _debug_file_exists_for_build_id(host, build_id), ( + f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/" + f".build-id/ matching postgres build-id {build_id} - the shipped " + f"_debug package may be out of sync with the shipped binary" + ) + + +def test_orioledb_library_build_id_matches_shipped_debug_symbols(host): + """Verify orioledb.so's build-id has a matching debug file, same as for + the postgres binary.""" + orioledb_so = _resolve_orioledb_lib(host, _resolve_postgres_binary(host)) + exists = run_ssh_command(host["ssh"], f"test -f {orioledb_so} && echo present") + if "present" not in exists["stdout"]: + pytest.skip("orioledb.so not present on this AMI (not an OrioleDB build)") + + build_id, readelf_output = _build_id_of(host, orioledb_so) + assert build_id, ( + f"Could not read a build-id from {orioledb_so}, readelf -n output:\n" + f"{readelf_output}" + ) + assert _debug_file_exists_for_build_id(host, build_id), ( + f"No debug file found under /var/lib/postgresql/.nix-profile/lib/debug/" + f".build-id/ matching orioledb.so build-id {build_id}" + ) + + +def test_gdb_resolves_postgres_source_via_shipped_src_package(host): + """Verify GDB can read actual source *content* for the installed postgres + binary via the shipped _src package, not just a known filename.""" + import re + + _skip_if_not_orioledb(host) + postgres_binary = _resolve_postgres_binary(host) + build_id, readelf_output = _build_id_of(host, postgres_binary) + assert build_id, ( + f"Could not read a build-id from {postgres_binary}, readelf -n output:\n" + f"{readelf_output}" + ) + prefix, rest = build_id[:2], build_id[2:] + debug_file = ( + f"/var/lib/postgresql/.nix-profile/lib/debug/.build-id/{prefix}/{rest}.debug" + ) + + # DW_AT_comp_dir is recorded per compilation unit, not once globally - + # PostgreSQL's recursive-Makefile build compiles each .c file from its + # own subdirectory (e.g. main.c from .../src/backend/main, a different + # object from a different subdirectory entirely), so grabbing the first + # DW_AT_comp_dir in the whole dump grabs whichever unrelated CU happens + # to appear first - not main.c's own. Find main.c's CU specifically and + # take its comp_dir. + main_c_comp_dir = run_ssh_command( + host["ssh"], + f"readelf --debug-dump=info {debug_file} 2>/dev/null | awk '" + "/DW_TAG_compile_unit/ { want=0 } " + "/DW_AT_name/ && /: main\\.c$/ { want=1 } " + "want && /DW_AT_comp_dir/ { print; exit }" + "' | grep -oE '/[^ ]+$'", + )["stdout"].strip() + assert main_c_comp_dir, ( + f"Could not determine main.c's DW_AT_comp_dir from {debug_file}" + ) + # main_c_comp_dir is main.c's own subdirectory (.../src/backend/main), not + # the shared build root - substituting that whole path would map main.c + # to '/main.c' instead of '/src/backend/main/main.c'. + # main.c's location in the postgres tree is fixed, so strip that known + # suffix to recover the actual build root shared by every CU. + suffix = "/src/backend/main" + assert main_c_comp_dir.endswith(suffix), ( + f"Expected main.c's comp_dir to end with {suffix!r}, got {main_c_comp_dir!r}" + ) + comp_dir = main_c_comp_dir[: -len(suffix)] + + result = run_ssh_command( + host["ssh"], + # cd into a directory the postgres user can actually read first: GDB's + # source search path includes '$cwd', and the SSH session's default + # cwd is the login user's home directory (e.g. /home/ubuntu), which + # postgres can't traverse - that alone is enough to make GDB report + # "Permission denied" on the bare filename before it ever tries the + # substitute-path'd absolute path. + "cd /var/lib/postgresql && sudo -u postgres gdb --batch -quiet " + "-ex 'set debug-file-directory /var/lib/postgresql/.nix-profile/lib/debug' " + f"-ex 'set substitute-path {comp_dir} /var/lib/postgresql/.nix-profile' " + f"-ex 'file {postgres_binary}' " + "-ex 'list main' " + "2>&1", + ) + assert result["succeeded"], f"gdb invocation failed: {result['stderr']}" + assert "No debugging symbols found" not in result["stdout"], ( + f"GDB could not find debug symbols for postgres:\n{result['stdout']}" + ) + assert not re.search(r"^\d+\tin /", result["stdout"], re.MULTILINE), ( + f"GDB fell back to the 'in ' placeholder, meaning it could not " + f"actually read the source file even with substitute-path set from " + f"{comp_dir} to /var/lib/postgresql/.nix-profile:\n{result['stdout']}" + ) + numbered_lines = re.findall(r"^\d+\t.+$", result["stdout"], re.MULTILINE) + assert len(numbered_lines) >= 3, ( + f"Expected 'list main' to print several lines of real source code " + f"content via the shipped _src package, got:\n{result['stdout']}" + ) + + +def test_coredump_processor_produces_diagnostic_bundle_and_deletes_raw_core(host): + """Verify that after a real Postgres backend crash, the + orioledb-coredump.path-triggered processor turns the raw core into a + diagnostic bundle and deletes the raw core.""" + unit_check = run_ssh_command( + host["ssh"], "systemctl list-unit-files orioledb-coredump.path --no-legend" + ) + if "orioledb-coredump.path" not in unit_check["stdout"]: + pytest.skip( + "coredump processor not installed on this AMI (not an OrioleDB build)" + ) + + before_bundles = set( + run_ssh_command( + host["ssh"], + "sudo find /var/lib/orioledb-coredumps/diagnostics -maxdepth 1 -type f", + )["stdout"].splitlines() + ) + + _crash_a_backend_and_wait_for_recovery(host) + + new_bundle = None + for _ in range(30): + sleep(2) + current = set( + run_ssh_command( + host["ssh"], + "sudo find /var/lib/orioledb-coredumps/diagnostics -maxdepth 1 -type f", + )["stdout"].splitlines() + ) + new = current - before_bundles + if new: + new_bundle = sorted(new)[0] + break + assert new_bundle, ( + "Expected a new diagnostic bundle under /var/lib/orioledb-coredumps/diagnostics " + "after the induced backend crash, but none appeared within 60s" + ) + + bundle_contents = run_ssh_command(host["ssh"], f"sudo cat {new_bundle}")["stdout"] + assert "gdb backtrace" in bundle_contents, ( + f"Expected the diagnostic bundle to contain a gdb backtrace section, got:\n" + f"{bundle_contents[:500]}" + ) + + remaining_cores = run_ssh_command( + host["ssh"], "sudo find /var/lib/systemd/coredump -maxdepth 1 -type f" + )["stdout"].strip() + assert remaining_cores == "", ( + f"Expected the raw core to be deleted after successful processing, but " + f"/var/lib/systemd/coredump still has:\n{remaining_cores}" + )