diff --git a/go/inst/instance_dao.go b/go/inst/instance_dao.go index 43d7b73b..6933fd6d 100644 --- a/go/inst/instance_dao.go +++ b/go/inst/instance_dao.go @@ -2694,6 +2694,29 @@ func ReadAllInstanceKeys() ([]InstanceKey, error) { return res, log.Errore(err) } +// ReadRecentlySeenInstanceKeyMap returns the set of instance keys whose last_seen is +// within the given recency window (hours). Used to protect freshly-discovered instances +// from being purged during raft snapshot restore before they've propagated into a snapshot. +func ReadRecentlySeenInstanceKeyMap(recencyHours uint) (*InstanceKeyMap, error) { + keys := NewInstanceKeyMap() + query := ` + select + hostname, port + from + database_instance + where + last_seen > NOW() - interval ? hour` + err := db.QueryOrchestrator(query, sqlutils.Args(recencyHours), func(m sqlutils.RowMap) error { + instanceKey, merr := NewResolveInstanceKey(m.GetString("hostname"), m.GetInt("port")) + if merr != nil { + return log.Errore(merr) + } + keys.AddKey(*instanceKey) + return nil + }) + return keys, log.Errore(err) +} + // ReadAllInstanceKeysMasterKeys func ReadAllMinimalInstances() ([]MinimalInstance, error) { res := []MinimalInstance{} diff --git a/go/logic/snapshot_data.go b/go/logic/snapshot_data.go index bd6e9cd0..b1327b63 100644 --- a/go/logic/snapshot_data.go +++ b/go/logic/snapshot_data.go @@ -22,6 +22,7 @@ import ( "encoding/json" "io" + "github.com/proxysql/orchestrator/go/config" "github.com/proxysql/orchestrator/go/db" "github.com/proxysql/orchestrator/go/inst" @@ -152,13 +153,26 @@ func (s *SnapshotDataCreatorApplier) Restore(rc io.ReadCloser) error { } discardedKeys := 0 - // Forget instances that were not in snapshot + // Forget instances that were not in snapshot. + // Guard: only forget an instance absent from the snapshot if it is ALSO stale + // locally (not seen within UnseenInstanceForgetHours). A freshly discovered + // instance may legitimately exist in our local backend but not yet be captured + // in the (older) snapshot we are restoring; deleting it here races with discovery + // and can wipe recent discoveries cluster-wide on restart/leader-change. Genuine + // decommissions age out and are removed both here and by ForgetLongUnseenInstances(), + // and explicit forgets arrive via the replicated "forget" command. existingKeys, _ := inst.ReadAllInstanceKeys() + recentlySeenKeys, _ := inst.ReadRecentlySeenInstanceKeyMap(config.Config.UnseenInstanceForgetHours) for _, existingKey := range existingKeys { - if !snapshotInstanceKeyMap.HasKey(existingKey) { - _ = inst.ForgetInstance(&existingKey) - discardedKeys++ + if snapshotInstanceKeyMap.HasKey(existingKey) { + continue } + if recentlySeenKeys.HasKey(existingKey) { + log.Debugf("raft snapshot restore: retaining recently-seen instance %+v absent from snapshot", existingKey) + continue + } + _ = inst.ForgetInstance(&existingKey) + discardedKeys++ } log.Debugf("raft snapshot restore: discarded %+v keys", discardedKeys) existingKeysMap := inst.NewInstanceKeyMap()