diff --git a/role_scripts/13/standby/remote-replica.sh b/role_scripts/13/standby/remote-replica.sh index 5a46fdb..8d1cfcf 100755 --- a/role_scripts/13/standby/remote-replica.sh +++ b/role_scripts/13/standby/remote-replica.sh @@ -20,27 +20,80 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,13 +126,64 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi fi +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 + fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf + # setup postgresql.conf touch /tmp/postgresql.conf @@ -147,10 +251,10 @@ fi # ****************** Recovery config ************************** echo "recovery_target_timeline = 'latest'" >>/tmp/postgresql.conf # primary_conninfo is used for streaming replication -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD'" >>/tmp/postgresql.conf fi echo "promote_trigger_file = '/run_scripts/tmp/pg-failover-trigger'" >>/tmp/postgresql.conf # [ name whose presence ends recovery] @@ -300,5 +404,29 @@ else fi fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/14/standby/remote-replica.sh b/role_scripts/14/standby/remote-replica.sh index cd68217..d2c78f8 100755 --- a/role_scripts/14/standby/remote-replica.sh +++ b/role_scripts/14/standby/remote-replica.sh @@ -20,27 +20,80 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,13 +126,64 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi fi +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 + fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf + # setup postgresql.conf touch /tmp/postgresql.conf @@ -147,10 +251,10 @@ fi # ****************** Recovery config ************************** echo "recovery_target_timeline = 'latest'" >>/tmp/postgresql.conf # primary_conninfo is used for streaming replication -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD'" >>/tmp/postgresql.conf fi echo "promote_trigger_file = '/run_scripts/tmp/pg-failover-trigger'" >>/tmp/postgresql.conf # [ name whose presence ends recovery] @@ -301,5 +405,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/15/standby/remote-replica.sh b/role_scripts/15/standby/remote-replica.sh index cd68217..d2c78f8 100755 --- a/role_scripts/15/standby/remote-replica.sh +++ b/role_scripts/15/standby/remote-replica.sh @@ -20,27 +20,80 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,13 +126,64 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi fi +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 + fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf + # setup postgresql.conf touch /tmp/postgresql.conf @@ -147,10 +251,10 @@ fi # ****************** Recovery config ************************** echo "recovery_target_timeline = 'latest'" >>/tmp/postgresql.conf # primary_conninfo is used for streaming replication -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD'" >>/tmp/postgresql.conf fi echo "promote_trigger_file = '/run_scripts/tmp/pg-failover-trigger'" >>/tmp/postgresql.conf # [ name whose presence ends recovery] @@ -301,5 +405,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/16/standby/remote-replica.sh b/role_scripts/16/standby/remote-replica.sh index b1dacdf..9b798ca 100755 --- a/role_scripts/16/standby/remote-replica.sh +++ b/role_scripts/16/standby/remote-replica.sh @@ -20,27 +20,80 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,13 +126,64 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi fi +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 + fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf + # setup postgresql.conf touch /tmp/postgresql.conf @@ -147,10 +251,10 @@ fi # ****************** Recovery config ************************** echo "recovery_target_timeline = 'latest'" >>/tmp/postgresql.conf # primary_conninfo is used for streaming replication -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD'" >>/tmp/postgresql.conf fi cat /run_scripts/role/postgresql.conf >>/tmp/postgresql.conf @@ -279,5 +383,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/17/standby/remote-replica.sh b/role_scripts/17/standby/remote-replica.sh index c9d00cd..5532956 100755 --- a/role_scripts/17/standby/remote-replica.sh +++ b/role_scripts/17/standby/remote-replica.sh @@ -20,27 +20,80 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,13 +126,64 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi fi +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 + fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf + # setup postgresql.conf touch /tmp/postgresql.conf @@ -152,10 +256,10 @@ if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then CONNINFO_DBNAME=" dbname=postgres" fi -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key$CONNINFO_DBNAME'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key$CONNINFO_DBNAME'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD$CONNINFO_DBNAME'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD$CONNINFO_DBNAME'" >>/tmp/postgresql.conf fi cat /run_scripts/role/postgresql.conf >>/tmp/postgresql.conf @@ -284,5 +388,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres diff --git a/role_scripts/18/standby/remote-replica.sh b/role_scripts/18/standby/remote-replica.sh index c9d00cd..5532956 100755 --- a/role_scripts/18/standby/remote-replica.sh +++ b/role_scripts/18/standby/remote-replica.sh @@ -20,27 +20,80 @@ export PRIMARY_PORT=${PRIMARY_PORT:-5432} echo "Running as Remote Replica" +# --------------------------------------------------------------------------- +# Upstream selection: cascading replication for a multi-replica remote replica. +# +# Ordinal 0 is the only pod that talks to the external source. Every other pod +# streams from ordinal 0 instead, so exactly one base backup and one WAL stream +# cross the WAN to the client's database rather than one per replica. +# +# The topology is also the one we want after cutover: promoting ordinal 0 turns +# the existing cascade into an ordinary primary-with-standbys tree, with no +# re-seed of the followers. +# --------------------------------------------------------------------------- +POD_ORDINAL="${HOSTNAME##*-}" +UPSTREAM_HOST="$PRIMARY_HOST" +UPSTREAM_PORT="$PRIMARY_PORT" +UPSTREAM_USER="$PRIMARY_USER_NAME" +UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" +UPSTREAM_SSL="${SOURCE_SSL:-OFF}" +UPSTREAM_SSL_MODE="${SOURCE_SSL_MODE:-disable}" +UPSTREAM_TLS_DIR="/tls/certs/remote" + +if [[ "${REPLICAS:-1}" -gt 1 && "$POD_ORDINAL" != "0" ]]; then + PETSET_BASE="${HOSTNAME%-*}" + # GOVERNING_SERVICE_DNS is supplied by the operator; the fallback keeps this + # script working against an operator that predates it. + UPSTREAM_HOST="${PETSET_BASE}-0.${GOVERNING_SERVICE_DNS:-${PETSET_BASE}-pods.${NAMESPACE}.svc}" + UPSTREAM_PORT="5432" + # Authenticate to ordinal 0 with the SOURCE's replication user, not the local + # one. Ordinal 0's pg_authid is a byte copy of the source's, so the source's + # user is the only one guaranteed to be present there AND to hold REPLICATION + # -- it is by definition the user the source DBA gave us to base-backup with. + # The local POSTGRES_USER carries no such guarantee: on a migration where the + # client hands over a restricted user instead of their superuser, it may not + # exist in the copied catalog at all. + # + # cert is the exception: clientcert=verify-full binds the certificate CN to + # the role name, and our client certificate is issued for this cluster's own + # user, so that is the only role it can present as. + if [[ "${CLIENT_AUTH_MODE:-md5}" == "cert" ]]; then + UPSTREAM_USER="${POSTGRES_USER:-postgres}" + UPSTREAM_PASSWORD="${POSTGRES_PASSWORD:-}" + else + UPSTREAM_USER="$PRIMARY_USER_NAME" + UPSTREAM_PASSWORD="${PRIMARY_PASSWORD:-}" + fi + # Peer-to-peer inside the cluster presents our own certs, not the source's. + UPSTREAM_SSL="${SSL:-OFF}" + UPSTREAM_SSL_MODE="${SSL_MODE:-disable}" + UPSTREAM_TLS_DIR="/tls/certs/client" + echo "cascade: ordinal $POD_ORDINAL follows $UPSTREAM_HOST:$UPSTREAM_PORT (not the external source)" +else + echo "cascade: ordinal $POD_ORDINAL follows the external source $UPSTREAM_HOST:$UPSTREAM_PORT" +fi + # set password ENV -export PGPASSWORD=${PRIMARY_PASSWORD:-} +export PGPASSWORD=${UPSTREAM_PASSWORD:-} # Waiting for running Postgres while true; do echo "Attempting pg_isready on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break else - pg_isready --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" --username=$PRIMARY_USER_NAME --timeout=2 &>/dev/null && break + pg_isready --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" --username=$UPSTREAM_USER --timeout=2 &>/dev/null && break fi sleep 2 done while true; do echo "Attempting query on primary" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d "dbname=postgres sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" --command="select now();" &>/dev/null && break + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d "dbname=postgres sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" --command="select now();" &>/dev/null && break else - psql -h "$PRIMARY_HOST" -p "$PRIMARY_PORT" --username=$PRIMARY_USER_NAME -d postgres --no-password --command="select now();" &>/dev/null && break + psql -h "$UPSTREAM_HOST" -p "$UPSTREAM_PORT" --username=$UPSTREAM_USER -d postgres --no-password --command="select now();" &>/dev/null && break fi sleep 2 @@ -73,13 +126,64 @@ if [[ ! -e "$PGDATA/PG_VERSION" ]]; then BASEBACKUP=pg_basebackup [[ "${TDE_ENABLED:-false}" == "true" ]] && BASEBACKUP=pg_tde_basebackup echo "pg_tde: seeding standby with '$BASEBACKUP' (TDE_ENABLED=${TDE_ENABLED:-false})" - if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" -d "sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key" + if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + "$BASEBACKUP" -Xs --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" -d "sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key" else - "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$PRIMARY_USER_NAME --progress --host="$PRIMARY_HOST" --port="$PRIMARY_PORT" + "$BASEBACKUP" -Xs --no-password --pgdata "$PGDATA" --username=$UPSTREAM_USER --progress --host="$UPSTREAM_HOST" --port="$UPSTREAM_PORT" fi fi +# --------------------------------------------------------------------------- +# Strip inherited recovery settings from postgresql.auto.conf. +# +# pg_basebackup copies the upstream's $PGDATA/postgresql.auto.conf verbatim, and +# Postgres reads that file AFTER postgresql.conf -- so anything ALTER SYSTEM'd +# upstream silently overrides the recovery configuration written below. On a +# repmgr- or patroni-managed source that means primary_conninfo and +# primary_slot_name pointing at the manager's own topology, and streaming then +# fails for a slot that was never ours: +# +# FATAL: could not start WAL streaming: +# ERROR: replication slot "repmgr_slot_1" does not exist +# +# retried forever, which is what a repeating 'started streaming WAL from +# primary at ' with no progress actually is. +# +# This runs on EVERY start, not only when seeding, because the seed is not the +# only thing that puts a foreign auto.conf in $PGDATA: the coordinator's own +# re-seed (pg_basebackup) and pg_rewind both copy from the upstream, and both +# leave PGDATA populated so the seed block above is skipped entirely. +# +# Only the keys KubeDB authors itself are removed; everything else -- a user's +# own ALTER SYSTEM -- is preserved, so this is safe to run unconditionally. +# --------------------------------------------------------------------------- +sanitize_auto_conf() { + local f="$PGDATA/postgresql.auto.conf" + [[ -s "$f" ]] || return 0 + local pattern='^[[:space:]]*(primary_conninfo|primary_slot_name|restore_command|recovery_target[a-z_]*|recovery_min_apply_delay|archive_mode|archive_command|archive_library)[[:space:]]*=' + if ! grep -Eq "$pattern" "$f"; then + return 0 + fi + echo "stripping inherited recovery settings from postgresql.auto.conf:" + grep -E "$pattern" "$f" | sed 's/password=[^ '"'"']*/password=/g; s/^/ | /' + # grep exits 1 when it selects no lines, which here means every line was a + # stripped key -- a normal outcome (an upstream managed entirely by repmgr or + # patroni has nothing else in the file), not an error. Under `set -e` the bare + # pipeline aborted the script at this point: postgres never started, the + # supervisor re-ran the role script, and it aborted again, forever -- with the + # container still reporting Ready. Only exit >1 is a real grep failure, and + # there we leave the file untouched rather than install a truncated one. + local rc=0 + grep -Ev "$pattern" "$f" >"$f.kubedb-tmp" || rc=$? + if ((rc > 1)); then + echo " ! grep failed (exit $rc); leaving postgresql.auto.conf unchanged" + rm -f "$f.kubedb-tmp" + return 0 + fi + mv "$f.kubedb-tmp" "$f" +} +sanitize_auto_conf + # setup postgresql.conf touch /tmp/postgresql.conf @@ -152,10 +256,10 @@ if [[ "$WAL_LIMIT_POLICY" == "ReplicationSlot" ]]; then CONNINFO_DBNAME=" dbname=postgres" fi -if [[ "${SOURCE_SSL:-0}" == "ON" ]]; then - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD sslmode=$SOURCE_SSL_MODE sslrootcert=/tls/certs/remote/ca.crt sslcert=/tls/certs/remote/client.crt sslkey=/tls/certs/remote/client.key$CONNINFO_DBNAME'" >>/tmp/postgresql.conf +if [[ "${UPSTREAM_SSL:-0}" == "ON" ]]; then + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD sslmode=$UPSTREAM_SSL_MODE sslrootcert=${UPSTREAM_TLS_DIR}/ca.crt sslcert=${UPSTREAM_TLS_DIR}/client.crt sslkey=${UPSTREAM_TLS_DIR}/client.key$CONNINFO_DBNAME'" >>/tmp/postgresql.conf else - echo "primary_conninfo = 'application_name=$HOSTNAME host=$PRIMARY_HOST port=$PRIMARY_PORT user=$PRIMARY_USER_NAME password=$PRIMARY_PASSWORD$CONNINFO_DBNAME'" >>/tmp/postgresql.conf + echo "primary_conninfo = 'application_name=$HOSTNAME host=$UPSTREAM_HOST port=$UPSTREAM_PORT user=$UPSTREAM_USER password=$UPSTREAM_PASSWORD$CONNINFO_DBNAME'" >>/tmp/postgresql.conf fi cat /run_scripts/role/postgresql.conf >>/tmp/postgresql.conf @@ -284,5 +388,29 @@ else fi +# --------------------------------------------------------------------------- +# Permit the source's replication user, not just the literal "postgres". +# +# Every replication rule emitted above names "postgres" explicitly. PostgreSQL +# does not match a replication connection against `all` in the DATABASE column, +# so when the source's replication user is anything else, a cascaded peer is +# refused before authentication even happens: +# +# FATAL: no pg_hba.conf entry for replication connection +# from host "10.x.x.x", user "migrator", no encryption +# +# Clone each postgres replication rule for that user rather than re-deriving the +# auth method per mode, so these rules cannot drift from the ones above. +# --------------------------------------------------------------------------- +if [[ -n "${PRIMARY_USER_NAME:-}" && "$PRIMARY_USER_NAME" != "postgres" ]]; then + echo "pg_hba: cloning replication rules for the source user '$PRIMARY_USER_NAME'" + awk -v u="$PRIMARY_USER_NAME" ' + $1 ~ /^host(ssl)?$/ && $2 == "replication" && $3 == "postgres" { + $3 = u; print + }' /tmp/pg_hba.conf >/tmp/pg_hba_extra.conf + cat /tmp/pg_hba_extra.conf >>/tmp/pg_hba.conf + rm -f /tmp/pg_hba_extra.conf +fi + mv /tmp/pg_hba.conf "$PGDATA/pg_hba.conf" exec postgres