From 0a3bcb0f67844b4c179479e8c23fa58e22ae3575 Mon Sep 17 00:00:00 2001 From: ShmuelOps Date: Mon, 3 Aug 2026 16:04:37 +0300 Subject: [PATCH] [redis-ha] Self-heal a masterless set instead of waiting for sentinel When the master is lost and its replacement rejoins as a replica behind the same announce address, sentinel keeps advertising that address as master and the set ends up with no master at all; sentinel aborts every failover with -NOGOODSLAVE because all candidates look stale. The split-brain container cannot help: it only fixes disagreement with sentinel, and it resolves sentinel through the headless service, which loses all endpoints once every pod is unready. Poll the announce-N services (publishNotReadyAddresses) for whether any member holds role:master. After MASTERLESS_CONFIRMATIONS consecutive observations of none, promote the reachable member with the highest replication offset, repoint the others, and reset the sentinels. The branch where sentinel names this pod as master but it runs as a replica routes into the same healer instead of reinit: reinit re-derives slaveof and shuts down, restarting the pod straight back into a replica while sentinel keeps naming it (#383). The misnamed pod is the fastest observer of the masterless state, and promotion stays offset-aware so a freshly replaced empty pod is never promoted merely because a stale sentinel record names it. Co-Authored-By: Claude Fable 5 Signed-off-by: ShmuelOps --- charts/redis-ha/Chart.yaml | 2 +- charts/redis-ha/templates/_configs.tpl | 172 ++++++++++++++++++++++++- 2 files changed, 171 insertions(+), 3 deletions(-) diff --git a/charts/redis-ha/Chart.yaml b/charts/redis-ha/Chart.yaml index d4e21a8..5c4e920 100644 --- a/charts/redis-ha/Chart.yaml +++ b/charts/redis-ha/Chart.yaml @@ -5,7 +5,7 @@ keywords: - redis - keyvalue - database -version: 4.39.0 +version: 4.40.0 appVersion: 8.8.0 description: This Helm chart provides a highly available Redis implementation with a master/slave configuration and uses Sentinel sidecars for failover management icon: https://img.icons8.com/external-tal-revivo-shadow-tal-revivo/24/external-redis-an-in-memory-data-structure-project-implementing-a-distributed-logo-shadow-tal-revivo.png diff --git a/charts/redis-ha/templates/_configs.tpl b/charts/redis-ha/templates/_configs.tpl index 450d129..993b977 100644 --- a/charts/redis-ha/templates/_configs.tpl +++ b/charts/redis-ha/templates/_configs.tpl @@ -493,6 +493,133 @@ set -e } + master_actual_role() { + set +e + if [ "$REDIS_PORT" -eq 0 ]; then + MASTER_ACTUAL_ROLE=$(redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "${MASTER}" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} info | grep role | sed 's/role://' | sed 's/\r//') + else + MASTER_ACTUAL_ROLE=$(redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "${MASTER}" -p "${REDIS_PORT}" info | grep role | sed 's/role://' | sed 's/\r//') + fi + set -e + } + + # Announce services publish not-ready addresses; the headless ${SERVICE} + # does not, so it stops resolving once every pod is unready. + REPLICA_COUNT={{ .Values.replicas }} + REDIS_ACTIVE_PORT="${REDIS_PORT}" + [ "$REDIS_PORT" -eq 0 ] && REDIS_ACTIVE_PORT="${REDIS_TLS_PORT}" + + announce_info() { + set +e + if [ "$REDIS_PORT" -eq 0 ]; then + redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$1" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} info replication 2>/dev/null | sed 's/\r//' + else + redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$1" -p "${REDIS_PORT}" info replication 2>/dev/null | sed 's/\r//' + fi + set -e + } + + announce_cmd() { + set +e + host="$1"; shift + if [ "$REDIS_PORT" -eq 0 ]; then + redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$host" -p "${REDIS_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@" + else + redis-cli {{ if .Values.auth }} -a "${AUTH}" --no-auth-warning{{ end }} -h "$host" -p "${REDIS_PORT}" "$@" + fi + set -e + } + + announce_sentinel_cmd() { + set +e + host="$1"; shift + if [ "$SENTINEL_PORT" -eq 0 ]; then + redis-cli {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} -h "$host" -p "${SENTINEL_TLS_PORT}" --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} "$@" + else + redis-cli {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} -h "$host" -p "${SENTINEL_PORT}" "$@" + fi + set -e + } + + # True if no member holds the master role. + masterless_now() { + _j=0 + while [ "$_j" -lt "$REPLICA_COUNT" ]; do + _r=$(announce_info "${SERVICE}-announce-${_j}" | awk -F: '/^role:/{print $2}') + [ "$_r" = "master" ] && return 1 + _j=$((_j + 1)) + done + return 0 + } + + # A masterless set never recovers on its own, so confirm it over seconds + # rather than over detection cycles. Any master seen aborts. + confirm_masterless_and_heal() { + _c=0 + while [ "$_c" -lt "$MASTERLESS_CONFIRMATIONS" ]; do + if ! masterless_now; then + echo "A member holds the master role; standing down after $_c/$MASTERLESS_CONFIRMATIONS confirmations." + return 1 + fi + _c=$((_c + 1)) + [ "$_c" -lt "$MASTERLESS_CONFIRMATIONS" ] && sleep "$MASTERLESS_CONFIRM_INTERVAL" + done + echo "No member held the master role across $MASTERLESS_CONFIRMATIONS consecutive checks." + promote_best_member + } + + # Promote the highest-offset member. Sentinel cannot: once every candidate + # looks stale it aborts with -failover-abort-no-good-slave / -NOGOODSLAVE. + promote_best_member() { + # Rescan: a liveness-killed redis is briefly unreachable. + _try=0 + while [ "$_try" -lt "$PROMOTE_SCAN_ATTEMPTS" ]; do + _best='' + _best_off=-1 + _i=0 + while [ "$_i" -lt "$REPLICA_COUNT" ]; do + _info=$(announce_info "${SERVICE}-announce-${_i}") + _role=$(echo "$_info" | awk -F: '/^role:/{print $2}') + if [ "$_role" = "master" ]; then + echo "A master is present at ${SERVICE}-announce-${_i}; no promotion needed." + return 1 + fi + _off=$(echo "$_info" | awk -F: '/_repl_offset:/{print $2; exit}') + case "$_off" in + ''|*[!0-9]*) ;; + *) if [ "$_off" -gt "$_best_off" ]; then _best_off="$_off"; _best="$_i"; fi ;; + esac + _i=$((_i + 1)) + done + [ -n "$_best" ] && break + _try=$((_try + 1)) + echo "Masterless, but no member answered (attempt $_try/$PROMOTE_SCAN_ATTEMPTS) — members may be restarting." + [ "$_try" -lt "$PROMOTE_SCAN_ATTEMPTS" ] && sleep "$MASTERLESS_CONFIRM_INTERVAL" + done + if [ -z "$_best" ]; then + echo "Masterless, but no member is reachable — not promoting." + return 1 + fi + _best_host="${SERVICE}-announce-${_best}" + _best_ip=$(getent hosts "$_best_host" | awk '{ print $1 }') + echo "ERROR: no member holds the master role. Promoting ${_best_host} (replication offset ${_best_off})." + announce_cmd "$_best_host" replicaof no one + _i=0 + while [ "$_i" -lt "$REPLICA_COUNT" ]; do + if [ "$_i" -ne "$_best" ] && [ -n "$_best_ip" ]; then + announce_cmd "${SERVICE}-announce-${_i}" replicaof "$_best_ip" "$REDIS_ACTIVE_PORT" + fi + _i=$((_i + 1)) + done + _i=0 + while [ "$_i" -lt "$REPLICA_COUNT" ]; do + announce_sentinel_cmd "${SERVICE}-announce-${_i}" sentinel reset "${MASTER_GROUP}" + _i=$((_i + 1)) + done + echo "Promotion complete: ${_best_host} is now master." + return 0 + } + identify_announce_ip while [ -z "${ANNOUNCE_IP}" ]; do @@ -503,6 +630,12 @@ QUORUM_FAIL_COUNT=0 MAX_QUORUM_FAILURES=${MAX_QUORUM_FAILURES:-5} + STALE_MASTER_COUNT=0 + MAX_STALE_MASTER_FAILURES=${MAX_STALE_MASTER_FAILURES:-3} + # Confirmations required before promoting, and the gap between them. + MASTERLESS_CONFIRMATIONS=${MASTERLESS_CONFIRMATIONS:-5} + MASTERLESS_CONFIRM_INTERVAL=${MASTERLESS_CONFIRM_INTERVAL:-5} + PROMOTE_SCAN_ATTEMPTS=${PROMOTE_SCAN_ATTEMPTS:-6} trap "exit 0" TERM while true; do @@ -521,8 +654,16 @@ redis_role echo "Redis role is $ROLE, expected role is master. No need to reinitialize." if [ "$ROLE" != "master" ]; then - echo "Redis role is $ROLE, expected role is master, reinitializing" - reinit + # Sentinel names this pod as master but it runs as a replica: + # reinit would re-derive `slaveof` and shutdown, restarting the + # pod straight back into a replica while sentinel keeps naming + # it (#383). This pod is also the fastest observer of the + # masterless state, so heal here instead of waiting for peers' + # stale-master counters. Promotion stays offset-aware — a + # freshly replaced (empty) pod behind a stale sentinel record + # must not be promoted just because sentinel names it. + echo "Redis role is $ROLE but sentinel names this pod as master; checking whether the set is masterless" + confirm_masterless_and_heal || true fi fi elif [ "${MASTER}" ]; then @@ -542,10 +683,37 @@ reinit fi fi + else + # Agreeing with sentinel is not enough: the named master may + # itself be a replica, leaving the set with no master at all. + master_actual_role + if [ -z "${MASTER_ACTUAL_ROLE}" ] || [ "${MASTER_ACTUAL_ROLE}" = "master" ]; then + # healthy, or unreachable (sentinel's own down detection handles that) + STALE_MASTER_COUNT=0 + else + STALE_MASTER_COUNT=$((STALE_MASTER_COUNT + 1)) + echo "WARNING: Sentinel names ${MASTER} as master but its actual role is '${MASTER_ACTUAL_ROLE}' (set may be masterless). Failure count: $STALE_MASTER_COUNT/$MAX_STALE_MASTER_FAILURES" + if [ "$STALE_MASTER_COUNT" -ge "$MAX_STALE_MASTER_FAILURES" ]; then + echo "ERROR: Sentinel-named master has not held the master role for $MAX_STALE_MASTER_FAILURES consecutive checks. Forcing a failover..." + if [ "$SENTINEL_PORT" -eq 0 ]; then + redis-cli -h "${SERVICE}" -p "${SENTINEL_TLS_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} --tls --cacert /tls-certs/{{ .Values.tls.caCertFile }} {{ if ne (default "yes" .Values.sentinel.authClients) "no"}} --cert /tls-certs/{{ .Values.tls.certFile }} --key /tls-certs/{{ .Values.tls.keyFile }}{{ end }} sentinel failover "${MASTER_GROUP}" || true + else + redis-cli -h "${SERVICE}" -p "${SENTINEL_PORT}" {{ if .Values.sentinel.auth }} -a "${SENTINELAUTH}" --no-auth-warning{{ end }} sentinel failover "${MASTER_GROUP}" || true + fi + # SENTINEL FAILOVER returns -NOGOODSLAVE when no + # candidate looks fresh; promote directly instead. + sleep {{ .Values.splitBrainDetection.retryInterval }} + confirm_masterless_and_heal || true + STALE_MASTER_COUNT=0 + fi + fi fi else QUORUM_FAIL_COUNT=$((QUORUM_FAIL_COUNT + 1)) echo "WARNING: Sentinel returned no master (quorum may be broken). Failure count: $QUORUM_FAIL_COUNT/$MAX_QUORUM_FAILURES" + # No master named is also what a masterless set looks like; the + # reset below still runs on its own schedule for real quorum loss. + confirm_masterless_and_heal || true if [ "$QUORUM_FAIL_COUNT" -ge "$MAX_QUORUM_FAILURES" ]; then echo "ERROR: Quorum broken for $MAX_QUORUM_FAILURES consecutive checks. Attempting sentinel reset..." if [ "$SENTINEL_PORT" -eq 0 ]; then