From 6206768b3feb7df058d597a10f5ceb896a6a4c7b Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Thu, 27 Aug 2026 20:02:48 +0600 Subject: [PATCH 1/4] Cascade a multi-replica remote replica, and drop the inherited auto.conf Two independent problems in the remote-replica seed path, both hit during a customer migration from a repmgr-managed source. 1. postgresql.auto.conf was inherited from the source. pg_basebackup copies $PGDATA/postgresql.auto.conf verbatim, and Postgres reads that file AFTER postgresql.conf -- so every setting the source had ALTER SYSTEM'd silently overrode the recovery configuration this script had just written. On a repmgr- or patroni-managed source that includes primary_conninfo and primary_slot_name, so the replica demanded a slot belonging to the manager's own standby: FATAL: could not start WAL streaming: ERROR: replication slot "repmgr_slot_1" does not exist repeating until the source recycled the WAL the seed still needed, at which point the base backup was dead and had to be retaken. Nothing in this repo ever touched that file. We pass no -R to pg_basebackup, so nothing we depend on lives in it. Truncate it once, at seed time, inside the fresh-PGDATA branch -- not on every start, which would discard a user's own later ALTER SYSTEM. The previous contents are echoed first, so an inherited setting leaves an audit line instead of vanishing. 2. Every replica seeded and streamed from the external source. All pods ran this script against PRIMARY_HOST, so a 3-replica remote replica took three base backups and held three WAL streams across the WAN against the customer's production database. Now only ordinal 0 follows the external source; every other pod follows ordinal 0. One base backup, one stream. Ordinal 0's catalog is a byte copy of the source's, so the operator-supplied credentials authenticate against it too, and peer connections present our own certs rather than the source's. The topology is also the one wanted after cutover: promoting ordinal 0 turns the existing cascade into an ordinary primary-with-standbys tree, with no re-seed of the followers. Verified: after promotion both followers stayed on zero base backups and simply followed the timeline switch to 2. GOVERNING_SERVICE_DNS supplies ordinal 0's address; the fallback derives it from HOSTNAME and NAMESPACE so the script still works against an operator that does not set it. Signed-off-by: souravbiswassanto --- role_scripts/13/standby/remote-replica.sh | 77 +++++++++++++++++++---- role_scripts/14/standby/remote-replica.sh | 77 +++++++++++++++++++---- role_scripts/15/standby/remote-replica.sh | 77 +++++++++++++++++++---- role_scripts/16/standby/remote-replica.sh | 77 +++++++++++++++++++---- role_scripts/17/standby/remote-replica.sh | 77 +++++++++++++++++++---- role_scripts/18/standby/remote-replica.sh | 77 +++++++++++++++++++---- 6 files changed, 384 insertions(+), 78 deletions(-) diff --git a/role_scripts/13/standby/remote-replica.sh b/role_scripts/13/standby/remote-replica.sh index 5a46fdb..87b8cb1 100755 --- a/role_scripts/13/standby/remote-replica.sh +++ b/role_scripts/13/standby/remote-replica.sh @@ -20,27 +20,66 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Ordinal 0's catalog is a byte copy of the source's, so the credentials the + # operator gave us -- which must match the source's -- authenticate there too. + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,11 +112,23 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" + fi + + # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and + # Postgres reads that file AFTER postgresql.conf -- so anything ALTER + # SYSTEM'd upstream silently overrides the recovery configuration written + # below, including the primary_conninfo and primary_slot_name an external + # manager such as repmgr or patroni leaves there. We pass no -R, so nothing + # we depend on lives in that file: start from empty. + if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then + echo "clearing inherited postgresql.auto.conf; it contained:" + sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" fi + : >"$PGDATA/postgresql.auto.conf" fi # setup postgresql.conf @@ -147,10 +198,10 @@ fi # ****************** Recovery config ************************** echo "recovery_target_timeline = 'latest'" >>/tmp/postgresql.conf # primary_conninfo is used for streaming replication -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD'" >>/tmp/postgresql.conf fi echo "promote_trigger_file = '/run_scripts/tmp/pg-failover-trigger'" >>/tmp/postgresql.conf # [ name whose presence ends recovery] diff --git a/role_scripts/14/standby/remote-replica.sh b/role_scripts/14/standby/remote-replica.sh index cd68217..c0948e3 100755 --- a/role_scripts/14/standby/remote-replica.sh +++ b/role_scripts/14/standby/remote-replica.sh @@ -20,27 +20,66 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Ordinal 0's catalog is a byte copy of the source's, so the credentials the + # operator gave us -- which must match the source's -- authenticate there too. + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,11 +112,23 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" + fi + + # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and + # Postgres reads that file AFTER postgresql.conf -- so anything ALTER + # SYSTEM'd upstream silently overrides the recovery configuration written + # below, including the primary_conninfo and primary_slot_name an external + # manager such as repmgr or patroni leaves there. We pass no -R, so nothing + # we depend on lives in that file: start from empty. + if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then + echo "clearing inherited postgresql.auto.conf; it contained:" + sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" fi + : >"$PGDATA/postgresql.auto.conf" fi # setup postgresql.conf @@ -147,10 +198,10 @@ fi # ****************** Recovery config ************************** echo "recovery_target_timeline = 'latest'" >>/tmp/postgresql.conf # primary_conninfo is used for streaming replication -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD'" >>/tmp/postgresql.conf fi echo "promote_trigger_file = '/run_scripts/tmp/pg-failover-trigger'" >>/tmp/postgresql.conf # [ name whose presence ends recovery] diff --git a/role_scripts/15/standby/remote-replica.sh b/role_scripts/15/standby/remote-replica.sh index cd68217..c0948e3 100755 --- a/role_scripts/15/standby/remote-replica.sh +++ b/role_scripts/15/standby/remote-replica.sh @@ -20,27 +20,66 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Ordinal 0's catalog is a byte copy of the source's, so the credentials the + # operator gave us -- which must match the source's -- authenticate there too. + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,11 +112,23 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" + fi + + # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and + # Postgres reads that file AFTER postgresql.conf -- so anything ALTER + # SYSTEM'd upstream silently overrides the recovery configuration written + # below, including the primary_conninfo and primary_slot_name an external + # manager such as repmgr or patroni leaves there. We pass no -R, so nothing + # we depend on lives in that file: start from empty. + if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then + echo "clearing inherited postgresql.auto.conf; it contained:" + sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" fi + : >"$PGDATA/postgresql.auto.conf" fi # setup postgresql.conf @@ -147,10 +198,10 @@ fi # ****************** Recovery config ************************** echo "recovery_target_timeline = 'latest'" >>/tmp/postgresql.conf # primary_conninfo is used for streaming replication -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD'" >>/tmp/postgresql.conf fi echo "promote_trigger_file = '/run_scripts/tmp/pg-failover-trigger'" >>/tmp/postgresql.conf # [ name whose presence ends recovery] diff --git a/role_scripts/16/standby/remote-replica.sh b/role_scripts/16/standby/remote-replica.sh index b1dacdf..86d8c5c 100755 --- a/role_scripts/16/standby/remote-replica.sh +++ b/role_scripts/16/standby/remote-replica.sh @@ -20,27 +20,66 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Ordinal 0's catalog is a byte copy of the source's, so the credentials the + # operator gave us -- which must match the source's -- authenticate there too. + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,11 +112,23 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" + fi + + # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and + # Postgres reads that file AFTER postgresql.conf -- so anything ALTER + # SYSTEM'd upstream silently overrides the recovery configuration written + # below, including the primary_conninfo and primary_slot_name an external + # manager such as repmgr or patroni leaves there. We pass no -R, so nothing + # we depend on lives in that file: start from empty. + if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then + echo "clearing inherited postgresql.auto.conf; it contained:" + sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" fi + : >"$PGDATA/postgresql.auto.conf" fi # setup postgresql.conf @@ -147,10 +198,10 @@ fi # ****************** Recovery config ************************** echo "recovery_target_timeline = 'latest'" >>/tmp/postgresql.conf # primary_conninfo is used for streaming replication -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD'" >>/tmp/postgresql.conf fi cat /run_scripts/role/postgresql.conf >>/tmp/postgresql.conf diff --git a/role_scripts/17/standby/remote-replica.sh b/role_scripts/17/standby/remote-replica.sh index c9d00cd..f2e540c 100755 --- a/role_scripts/17/standby/remote-replica.sh +++ b/role_scripts/17/standby/remote-replica.sh @@ -20,27 +20,66 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Ordinal 0's catalog is a byte copy of the source's, so the credentials the + # operator gave us -- which must match the source's -- authenticate there too. + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,11 +112,23 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" + fi + + # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and + # Postgres reads that file AFTER postgresql.conf -- so anything ALTER + # SYSTEM'd upstream silently overrides the recovery configuration written + # below, including the primary_conninfo and primary_slot_name an external + # manager such as repmgr or patroni leaves there. We pass no -R, so nothing + # we depend on lives in that file: start from empty. + if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then + echo "clearing inherited postgresql.auto.conf; it contained:" + sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" fi + : >"$PGDATA/postgresql.auto.conf" fi # setup postgresql.conf @@ -152,10 +203,10 @@ if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then CONNINFO_DBNAME=" dbname=postgres" fi -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key$CONNINFO_DBNAME'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key$CONNINFO_DBNAME'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD$CONNINFO_DBNAME'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD$CONNINFO_DBNAME'" >>/tmp/postgresql.conf fi cat /run_scripts/role/postgresql.conf >>/tmp/postgresql.conf diff --git a/role_scripts/18/standby/remote-replica.sh b/role_scripts/18/standby/remote-replica.sh index c9d00cd..f2e540c 100755 --- a/role_scripts/18/standby/remote-replica.sh +++ b/role_scripts/18/standby/remote-replica.sh @@ -20,27 +20,66 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Ordinal 0's catalog is a byte copy of the source's, so the credentials the + # operator gave us -- which must match the source's -- authenticate there too. + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,11 +112,23 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" + fi + + # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and + # Postgres reads that file AFTER postgresql.conf -- so anything ALTER + # SYSTEM'd upstream silently overrides the recovery configuration written + # below, including the primary_conninfo and primary_slot_name an external + # manager such as repmgr or patroni leaves there. We pass no -R, so nothing + # we depend on lives in that file: start from empty. + if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then + echo "clearing inherited postgresql.auto.conf; it contained:" + sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" fi + : >"$PGDATA/postgresql.auto.conf" fi # setup postgresql.conf @@ -152,10 +203,10 @@ if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then CONNINFO_DBNAME=" dbname=postgres" fi -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key$CONNINFO_DBNAME'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key$CONNINFO_DBNAME'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD$CONNINFO_DBNAME'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD$CONNINFO_DBNAME'" >>/tmp/postgresql.conf fi cat /run_scripts/role/postgresql.conf >>/tmp/postgresql.conf From ffb06dd7462f5358c11ad4c0bee7ffd9e73f4c92 Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Thu, 27 Aug 2026 21:22:39 +0600 Subject: [PATCH 2/4] Sanitize the inherited auto.conf on every start, not just at seed The truncation added in the previous commit ran only inside the fresh-PGDATA branch, which is not the only way a foreign postgresql.auto.conf reaches $PGDATA. The coordinator's own re-seed (pg_basebackup in pg_basebackup.go) and pg_rewind both copy from the upstream and both leave PGDATA populated, so the seed block -- and with it the truncation -- is skipped on the restart that follows. The upstream's primary_conninfo and primary_slot_name were then back in force, silently overriding what this script writes into postgresql.conf, because Postgres reads auto.conf second. That is a loop, not just a missed cleanup: streaming fails for a slot that was never ours, the coordinator sees the pod absent from the upstream's pg_stat_replication, re-seeds, and the re-seed restores the same auto.conf. Run the sanitisation unconditionally instead, and make it selective rather than truncating: strip only the keys KubeDB authors itself (primary_conninfo, primary_slot_name, restore_command, recovery_target*, recovery_min_apply_delay, archive_mode/command/library) and leave everything else alone. That is safe to run on every start -- a user's own ALTER SYSTEM survives -- so it no longer depends on the fresh-seed guard, and it covers every path that can seed rather than just the one. Passwords in the reported primary_conninfo are redacted in the log line. Signed-off-by: souravbiswassanto --- role_scripts/13/standby/remote-replica.sh | 48 +++++++++++++++++------ role_scripts/14/standby/remote-replica.sh | 48 +++++++++++++++++------ role_scripts/15/standby/remote-replica.sh | 48 +++++++++++++++++------ role_scripts/16/standby/remote-replica.sh | 48 +++++++++++++++++------ role_scripts/17/standby/remote-replica.sh | 48 +++++++++++++++++------ role_scripts/18/standby/remote-replica.sh | 48 +++++++++++++++++------ 6 files changed, 222 insertions(+), 66 deletions(-) diff --git a/role_scripts/13/standby/remote-replica.sh b/role_scripts/13/standby/remote-replica.sh index 87b8cb1..f33fcc4 100755 --- a/role_scripts/13/standby/remote-replica.sh +++ b/role_scripts/13/standby/remote-replica.sh @@ -117,19 +117,45 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then else "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi +fi - # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and - # Postgres reads that file AFTER postgresql.conf -- so anything ALTER - # SYSTEM'd upstream silently overrides the recovery configuration written - # below, including the primary_conninfo and primary_slot_name an external - # manager such as repmgr or patroni leaves there. We pass no -R, so nothing - # we depend on lives in that file: start from empty. - if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then - echo "clearing inherited postgresql.auto.conf; it contained:" - sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 fi - : >"$PGDATA/postgresql.auto.conf" -fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf # setup postgresql.conf touch /tmp/postgresql.conf diff --git a/role_scripts/14/standby/remote-replica.sh b/role_scripts/14/standby/remote-replica.sh index c0948e3..59934e2 100755 --- a/role_scripts/14/standby/remote-replica.sh +++ b/role_scripts/14/standby/remote-replica.sh @@ -117,19 +117,45 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then else "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi +fi - # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and - # Postgres reads that file AFTER postgresql.conf -- so anything ALTER - # SYSTEM'd upstream silently overrides the recovery configuration written - # below, including the primary_conninfo and primary_slot_name an external - # manager such as repmgr or patroni leaves there. We pass no -R, so nothing - # we depend on lives in that file: start from empty. - if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then - echo "clearing inherited postgresql.auto.conf; it contained:" - sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 fi - : >"$PGDATA/postgresql.auto.conf" -fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf # setup postgresql.conf touch /tmp/postgresql.conf diff --git a/role_scripts/15/standby/remote-replica.sh b/role_scripts/15/standby/remote-replica.sh index c0948e3..59934e2 100755 --- a/role_scripts/15/standby/remote-replica.sh +++ b/role_scripts/15/standby/remote-replica.sh @@ -117,19 +117,45 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then else "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi +fi - # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and - # Postgres reads that file AFTER postgresql.conf -- so anything ALTER - # SYSTEM'd upstream silently overrides the recovery configuration written - # below, including the primary_conninfo and primary_slot_name an external - # manager such as repmgr or patroni leaves there. We pass no -R, so nothing - # we depend on lives in that file: start from empty. - if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then - echo "clearing inherited postgresql.auto.conf; it contained:" - sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 fi - : >"$PGDATA/postgresql.auto.conf" -fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf # setup postgresql.conf touch /tmp/postgresql.conf diff --git a/role_scripts/16/standby/remote-replica.sh b/role_scripts/16/standby/remote-replica.sh index 86d8c5c..754d405 100755 --- a/role_scripts/16/standby/remote-replica.sh +++ b/role_scripts/16/standby/remote-replica.sh @@ -117,19 +117,45 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then else "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi +fi - # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and - # Postgres reads that file AFTER postgresql.conf -- so anything ALTER - # SYSTEM'd upstream silently overrides the recovery configuration written - # below, including the primary_conninfo and primary_slot_name an external - # manager such as repmgr or patroni leaves there. We pass no -R, so nothing - # we depend on lives in that file: start from empty. - if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then - echo "clearing inherited postgresql.auto.conf; it contained:" - sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 fi - : >"$PGDATA/postgresql.auto.conf" -fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf # setup postgresql.conf touch /tmp/postgresql.conf diff --git a/role_scripts/17/standby/remote-replica.sh b/role_scripts/17/standby/remote-replica.sh index f2e540c..9e02677 100755 --- a/role_scripts/17/standby/remote-replica.sh +++ b/role_scripts/17/standby/remote-replica.sh @@ -117,19 +117,45 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then else "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi +fi - # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and - # Postgres reads that file AFTER postgresql.conf -- so anything ALTER - # SYSTEM'd upstream silently overrides the recovery configuration written - # below, including the primary_conninfo and primary_slot_name an external - # manager such as repmgr or patroni leaves there. We pass no -R, so nothing - # we depend on lives in that file: start from empty. - if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then - echo "clearing inherited postgresql.auto.conf; it contained:" - sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 fi - : >"$PGDATA/postgresql.auto.conf" -fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf # setup postgresql.conf touch /tmp/postgresql.conf diff --git a/role_scripts/18/standby/remote-replica.sh b/role_scripts/18/standby/remote-replica.sh index f2e540c..9e02677 100755 --- a/role_scripts/18/standby/remote-replica.sh +++ b/role_scripts/18/standby/remote-replica.sh @@ -117,19 +117,45 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then else "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi +fi - # pg_basebackup copies the upstream's postgresql.auto.conf verbatim, and - # Postgres reads that file AFTER postgresql.conf -- so anything ALTER - # SYSTEM'd upstream silently overrides the recovery configuration written - # below, including the primary_conninfo and primary_slot_name an external - # manager such as repmgr or patroni leaves there. We pass no -R, so nothing - # we depend on lives in that file: start from empty. - if [[ -s "$PGDATA/postgresql.auto.conf" ]]; then - echo "clearing inherited postgresql.auto.conf; it contained:" - sed 's/^/ | /' "$PGDATA/postgresql.auto.conf" +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 fi - : >"$PGDATA/postgresql.auto.conf" -fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf # setup postgresql.conf touch /tmp/postgresql.conf From c73bd0ae149b92abb908e483ce554ff33e57b632 Mon Sep 17 00:00:00 2001 From: souravbiswassanto Date: Thu, 27 Aug 2026 22:07:42 +0600 Subject: [PATCH 3/4] Authenticate the cascade leg as the source's replication user The cascade leg connected to ordinal 0 as the local POSTGRES_USER, and the pg_hba.conf this script generates names the literal user "postgres" in every replication rule. Two problems follow, and neither shows up in a lab where the client happens to hand over their superuser. First, the dependency is on the wrong role. Ordinal 0's pg_authid is a byte copy of the source's, so the role that definitionally exists there with REPLICATION is the one the source DBA created for us to base-backup with -- PRIMARY_USER_NAME. "postgres" carries no such guarantee: a client who hands over a restricted user instead of their superuser may have no postgres role at all, which is why the migration runbook already tells them to check for it and CREATE ROLE postgres if it is missing. The cascade should not depend on that remediation having happened. Second, `all` in pg_hba's DATABASE column does not match a replication connection, so once the connecting user is anything other than "postgres" there is no matching rule and the peer is refused before authentication: FATAL: no pg_hba.conf entry for replication connection from host "10.42.0.230", user "migrator", no encryption Use PRIMARY_USER_NAME for the cascade leg, and clone each "postgres" replication rule for that user rather than re-deriving the auth method per CLIENT_AUTH_MODE, so the two sets cannot drift. cert mode keeps the local user: clientcert=verify-full binds the certificate CN to the role name, and our client certificate is issued for this cluster's own user, so that is the only role it can present as. Verified live on a three-replica cascade whose source user is 'migrator': pg_hba gains the two cloned rules with the scram method preserved, both followers' primary_conninfo carries user=migrator, pg_stat_replication shows all three streaming as migrator, and the physical replication connection that previously returned the FATAL above now succeeds. Signed-off-by: souravbiswassanto --- role_scripts/13/standby/remote-replica.sh | 46 +++++++++++++++++++++-- role_scripts/14/standby/remote-replica.sh | 46 +++++++++++++++++++++-- role_scripts/15/standby/remote-replica.sh | 46 +++++++++++++++++++++-- role_scripts/16/standby/remote-replica.sh | 46 +++++++++++++++++++++-- role_scripts/17/standby/remote-replica.sh | 46 +++++++++++++++++++++-- role_scripts/18/standby/remote-replica.sh | 46 +++++++++++++++++++++-- 6 files changed, 252 insertions(+), 24 deletions(-) diff --git a/role_scripts/13/standby/remote-replica.sh b/role_scripts/13/standby/remote-replica.sh index f33fcc4..729d43f 100755 --- a/role_scripts/13/standby/remote-replica.sh +++ b/role_scripts/13/standby/remote-replica.sh @@ -46,10 +46,24 @@ if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then # script working against an operator that predates it. UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" UPSTREAM_PORT="5432" - # Ordinal 0's catalog is a byte copy of the source's, so the credentials the - # operator gave us -- which must match the source's -- authenticate there too. - UPSTREAM_USER="${POSTGRES_USER:-postgres}" - UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi # Peer-to-peer inside the cluster presents our own certs, not the source's. UPSTREAM_SSL="${SSL:-OFF}" UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" @@ -377,5 +391,29 @@ else fi fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/14/standby/remote-replica.sh b/role_scripts/14/standby/remote-replica.sh index 59934e2..101f3a1 100755 --- a/role_scripts/14/standby/remote-replica.sh +++ b/role_scripts/14/standby/remote-replica.sh @@ -46,10 +46,24 @@ if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then # script working against an operator that predates it. UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" UPSTREAM_PORT="5432" - # Ordinal 0's catalog is a byte copy of the source's, so the credentials the - # operator gave us -- which must match the source's -- authenticate there too. - UPSTREAM_USER="${POSTGRES_USER:-postgres}" - UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi # Peer-to-peer inside the cluster presents our own certs, not the source's. UPSTREAM_SSL="${SSL:-OFF}" UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" @@ -378,5 +392,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/15/standby/remote-replica.sh b/role_scripts/15/standby/remote-replica.sh index 59934e2..101f3a1 100755 --- a/role_scripts/15/standby/remote-replica.sh +++ b/role_scripts/15/standby/remote-replica.sh @@ -46,10 +46,24 @@ if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then # script working against an operator that predates it. UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" UPSTREAM_PORT="5432" - # Ordinal 0's catalog is a byte copy of the source's, so the credentials the - # operator gave us -- which must match the source's -- authenticate there too. - UPSTREAM_USER="${POSTGRES_USER:-postgres}" - UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi # Peer-to-peer inside the cluster presents our own certs, not the source's. UPSTREAM_SSL="${SSL:-OFF}" UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" @@ -378,5 +392,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/16/standby/remote-replica.sh b/role_scripts/16/standby/remote-replica.sh index 754d405..78b0108 100755 --- a/role_scripts/16/standby/remote-replica.sh +++ b/role_scripts/16/standby/remote-replica.sh @@ -46,10 +46,24 @@ if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then # script working against an operator that predates it. UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" UPSTREAM_PORT="5432" - # Ordinal 0's catalog is a byte copy of the source's, so the credentials the - # operator gave us -- which must match the source's -- authenticate there too. - UPSTREAM_USER="${POSTGRES_USER:-postgres}" - UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi # Peer-to-peer inside the cluster presents our own certs, not the source's. UPSTREAM_SSL="${SSL:-OFF}" UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" @@ -356,5 +370,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/17/standby/remote-replica.sh b/role_scripts/17/standby/remote-replica.sh index 9e02677..81e3bcb 100755 --- a/role_scripts/17/standby/remote-replica.sh +++ b/role_scripts/17/standby/remote-replica.sh @@ -46,10 +46,24 @@ if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then # script working against an operator that predates it. UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" UPSTREAM_PORT="5432" - # Ordinal 0's catalog is a byte copy of the source's, so the credentials the - # operator gave us -- which must match the source's -- authenticate there too. - UPSTREAM_USER="${POSTGRES_USER:-postgres}" - UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi # Peer-to-peer inside the cluster presents our own certs, not the source's. UPSTREAM_SSL="${SSL:-OFF}" UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" @@ -361,5 +375,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/18/standby/remote-replica.sh b/role_scripts/18/standby/remote-replica.sh index 9e02677..81e3bcb 100755 --- a/role_scripts/18/standby/remote-replica.sh +++ b/role_scripts/18/standby/remote-replica.sh @@ -46,10 +46,24 @@ if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then # script working against an operator that predates it. UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" UPSTREAM_PORT="5432" - # Ordinal 0's catalog is a byte copy of the source's, so the credentials the - # operator gave us -- which must match the source's -- authenticate there too. - UPSTREAM_USER="${POSTGRES_USER:-postgres}" - UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi # Peer-to-peer inside the cluster presents our own certs, not the source's. UPSTREAM_SSL="${SSL:-OFF}" UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" @@ -361,5 +375,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres From aa1cfb3278a33a0ab7622d2abcf131e40c0f5e78 Mon Sep 17 00:00:00 2001 From: Tamal Saha Date: Fri, 28 Aug 2026 17:00:51 +0600 Subject: [PATCH 4/4] Do not abort the sanitiser when every auto.conf line is stripped sanitize_auto_conf ran `grep -Ev "$pattern" "$f"` unguarded under `set -eou pipefail`. grep exits 1 when it selects no lines, which happens whenever the inherited postgresql.auto.conf consists *only* of keys we strip -- exactly what an upstream managed entirely by repmgr or patroni looks like, since primary_conninfo and primary_slot_name may be the whole file. The failure is silent and total. The role script dies immediately after logging what it was about to strip, so the file is left unmodified with the foreign primary_conninfo still in it; postgres never starts; the supervisor notices the server is down and re-runs the role script, which dies at the same line, forever. Meanwhile both containers report Ready with zero restarts, so nothing at the pod level indicates a problem -- only the Postgres CR going Critical does. Observed on a remote replica whose postgresql.auto.conf held only primary_conninfo and primary_slot_name: stripping inherited recovery settings from postgresql.auto.conf: | primary_conninfo = 'host=old-manager user=repmgr' | primary_slot_name = 'repmgr_slot_1' removing the initial scripts as server is not running ... running the initial script ... Running as Remote Replica ... same three lines again, 4 times in 2 minutes A PostgreSQL-written auto.conf normally carries ALTER SYSTEM's two comment lines, which never match the pattern and so keep grep at exit 0 -- that is why the common path survives. Any upstream presenting a headerless file hits this. Treat exit 1 as the normal "nothing left to keep" outcome and write the resulting empty file. Only exit >1 is a real grep failure, and there the original is left untouched rather than replaced by a truncated temp file. Signed-off-by: Tamal Saha --- role_scripts/13/standby/remote-replica.sh | 15 ++++++++++++++- role_scripts/14/standby/remote-replica.sh | 15 ++++++++++++++- role_scripts/15/standby/remote-replica.sh | 15 ++++++++++++++- role_scripts/16/standby/remote-replica.sh | 15 ++++++++++++++- role_scripts/17/standby/remote-replica.sh | 15 ++++++++++++++- role_scripts/18/standby/remote-replica.sh | 15 ++++++++++++++- 6 files changed, 84 insertions(+), 6 deletions(-) diff --git a/role_scripts/13/standby/remote-replica.sh b/role_scripts/13/standby/remote-replica.sh index 729d43f..8d1cfcf 100755 --- a/role_scripts/13/standby/remote-replica.sh +++ b/role_scripts/13/standby/remote-replica.sh @@ -166,7 +166,20 @@ sanitize_auto_conf() { fi echo "stripping inherited recovery settings from postgresql.auto.conf:" grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' - grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi mv "$f.kubedb-tmp" "$f" } sanitize_auto_conf diff --git a/role_scripts/14/standby/remote-replica.sh b/role_scripts/14/standby/remote-replica.sh index 101f3a1..d2c78f8 100755 --- a/role_scripts/14/standby/remote-replica.sh +++ b/role_scripts/14/standby/remote-replica.sh @@ -166,7 +166,20 @@ sanitize_auto_conf() { fi echo "stripping inherited recovery settings from postgresql.auto.conf:" grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' - grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi mv "$f.kubedb-tmp" "$f" } sanitize_auto_conf diff --git a/role_scripts/15/standby/remote-replica.sh b/role_scripts/15/standby/remote-replica.sh index 101f3a1..d2c78f8 100755 --- a/role_scripts/15/standby/remote-replica.sh +++ b/role_scripts/15/standby/remote-replica.sh @@ -166,7 +166,20 @@ sanitize_auto_conf() { fi echo "stripping inherited recovery settings from postgresql.auto.conf:" grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' - grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi mv "$f.kubedb-tmp" "$f" } sanitize_auto_conf diff --git a/role_scripts/16/standby/remote-replica.sh b/role_scripts/16/standby/remote-replica.sh index 78b0108..9b798ca 100755 --- a/role_scripts/16/standby/remote-replica.sh +++ b/role_scripts/16/standby/remote-replica.sh @@ -166,7 +166,20 @@ sanitize_auto_conf() { fi echo "stripping inherited recovery settings from postgresql.auto.conf:" grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' - grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi mv "$f.kubedb-tmp" "$f" } sanitize_auto_conf diff --git a/role_scripts/17/standby/remote-replica.sh b/role_scripts/17/standby/remote-replica.sh index 81e3bcb..5532956 100755 --- a/role_scripts/17/standby/remote-replica.sh +++ b/role_scripts/17/standby/remote-replica.sh @@ -166,7 +166,20 @@ sanitize_auto_conf() { fi echo "stripping inherited recovery settings from postgresql.auto.conf:" grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' - grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi mv "$f.kubedb-tmp" "$f" } sanitize_auto_conf diff --git a/role_scripts/18/standby/remote-replica.sh b/role_scripts/18/standby/remote-replica.sh index 81e3bcb..5532956 100755 --- a/role_scripts/18/standby/remote-replica.sh +++ b/role_scripts/18/standby/remote-replica.sh @@ -166,7 +166,20 @@ sanitize_auto_conf() { fi echo "stripping inherited recovery settings from postgresql.auto.conf:" grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' - grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi mv "$f.kubedb-tmp" "$f" } sanitize_auto_conf