Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 7 additions & 10 deletions crates/engram-coordinator/src/api/hosts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,7 @@ impl HostView {
) -> Self {
let mut ready_image_digests = row.ready_images.clone();
ready_image_digests.sort();
let allocatable_mib = row.utilization.allocatable_mib;
let reserved_mib = reserved.mem_mib.max(0) as u64;
let cpu_budget = engram_core::types::host::host_cpu_budget(row.total_vcpus).max(0) as u64;
let reserved_vcpus = reserved.vcpus.max(0) as u64;
let room = engram_core::types::host::HostHeadroom::of(&row, reserved);
let failing_capabilities = row
.capabilities
.failing_capabilities()
Expand Down Expand Up @@ -141,13 +138,13 @@ impl HostView {
pool_data_total_mib: row.utilization.pool_data_total_mib,
pool_metadata_free_mib: row.utilization.pool_metadata_free_mib,
pool_metadata_total_mib: row.utilization.pool_metadata_total_mib,
allocatable_mib,
reserved_mib,
free_mib: allocatable_mib.saturating_sub(reserved_mib),
allocatable_mib: room.allocatable_mib.max(0) as u64,
reserved_mib: room.reserved.mem_mib.max(0) as u64,
free_mib: room.free_mib().max(0) as u64,
total_vcpus: row.total_vcpus,
cpu_budget_vcpus: cpu_budget,
reserved_vcpus,
free_vcpus: cpu_budget.saturating_sub(reserved_vcpus),
cpu_budget_vcpus: room.cpu_budget.max(0) as u64,
reserved_vcpus: room.reserved.vcpus.max(0) as u64,
free_vcpus: room.free_vcpus().unwrap_or(0).max(0) as u64,
last_heartbeat_at: row.last_heartbeat_at,
failing_capabilities,
fc_snapshot_version,
Expand Down
512 changes: 419 additions & 93 deletions crates/engram-coordinator/src/idle_evictor.rs

Large diffs are not rendered by default.

65 changes: 24 additions & 41 deletions crates/engram-coordinator/src/placement.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use std::time::Duration;
use chrono::{DateTime, Utc};
use engram_core::traits::{HostClient, MetadataStore, SessionFence};
use engram_core::types::host::{
CapStatus, HostRecord, HostStatus, ReservedBudget, HOST_POOL_METADATA_FLOOR_MIB,
CapStatus, HostHeadroom, HostRecord, HostStatus, ReservedBudget, HOST_POOL_METADATA_FLOOR_MIB,
};
use engram_core::types::snapshot::SnapshotMetadata;
use engram_core::{HostId, SandboxError, SandboxId};
Expand Down Expand Up @@ -558,25 +558,22 @@ pub fn pick_from(
}
let need_mib = ctx.memory_mib.unwrap_or(0) as i64;
let need_vcpus = ctx.cpu_budget_vcpus.unwrap_or(0) as i64;
let headroom = |id: HostId| -> Option<HostHeadroom> {
let h = hosts.iter().find(|h| h.id == id)?;
Some(HostHeadroom::of(
h,
reserved.get(&id).copied().unwrap_or_default(),
))
};
// Free RAM for `id`: None ⇒ unmeasured (treated as "fits" softly).
let free_mib_of = |id: HostId| -> Option<i64> {
let h = hosts.iter().find(|h| h.id == id)?;
let alloc = h.utilization.allocatable_mib as i64;
if alloc <= 0 {
return None;
}
Some(alloc - reserved.get(&id).map(|r| r.mem_mib).unwrap_or(0))
headroom(id)
.filter(HostHeadroom::measured)
.map(|r| r.free_mib())
};
// CPU fit for `id`: a host with budget 0 (unreported) doesn't gate.
// CPU fit for `id`: a host with no reported cores doesn't gate.
let cpu_fits = |id: HostId| -> bool {
let Some(h) = hosts.iter().find(|h| h.id == id) else {
return false;
};
let budget = engram_core::types::host::host_cpu_budget(h.total_vcpus);
if budget <= 0 {
return true;
}
budget - reserved.get(&id).map(|r| r.vcpus).unwrap_or(0) >= need_vcpus
headroom(id).is_some_and(|r| r.free_vcpus().is_none_or(|free| free >= need_vcpus))
};
// Tier 0 (ADR 0078): authoritative snapshot-host affinity. If the
// host that holds this snapshot's chunks is alive, schedulable, and
Expand Down Expand Up @@ -734,25 +731,12 @@ pub async fn placement_preview(
let (hosts, reserved) = hosts_and_reserved(meta).await?;
let ranked = rank_hosts(&hosts, ctx, now, placement_ttl());
Ok(ranked.hosts.iter().any(|id| {
let Some(h) = hosts.iter().find(|h| h.id == *id) else {
return false;
};
let alloc = h.utilization.allocatable_mib as i64;
if alloc <= 0 {
return true; // unmeasured → soft fallback fits
}
let free_mib = alloc - reserved.get(id).map(|r| r.mem_mib).unwrap_or(0);
if free_mib < mem_mib {
return false;
}
let cpu_budget = engram_core::types::host::host_cpu_budget(h.total_vcpus);
if cpu_budget > 0 {
let free_vcpus = cpu_budget - reserved.get(id).map(|r| r.vcpus).unwrap_or(0);
if free_vcpus < cpu_vcpus {
return false;
}
}
true
hosts.iter().find(|h| h.id == *id).is_some_and(|h| {
let room = HostHeadroom::of(h, reserved.get(id).copied().unwrap_or_default());
// The placement transaction takes an unmeasured host as its
// last resort without a CPU check.
!room.measured() || room.fits(mem_mib, cpu_vcpus)
})
}))
}

Expand Down Expand Up @@ -1198,12 +1182,11 @@ pub async fn fleet_snapshot(
}
if host_counts_as_capacity(h, now, ttl) {
m.schedulable_hosts += 1;
m.total_mib += h.utilization.allocatable_mib;
let cpu_budget = engram_core::types::host::host_cpu_budget(h.total_vcpus);
m.total_vcpus += cpu_budget as u64;
let r = reserved.get(&h.id).copied().unwrap_or_default();
m.free_vcpus += (cpu_budget - r.vcpus).max(0) as u64;
m.free_mib += (h.utilization.allocatable_mib as i64 - r.mem_mib).max(0) as u64;
let room = HostHeadroom::of(h, reserved.get(&h.id).copied().unwrap_or_default());
m.total_mib += room.allocatable_mib.max(0) as u64;
m.total_vcpus += room.cpu_budget.max(0) as u64;
m.free_vcpus += room.free_vcpus().unwrap_or(0).max(0) as u64;
m.free_mib += room.free_mib().max(0) as u64;
}
}
Ok(m)
Expand Down
27 changes: 27 additions & 0 deletions crates/engram-coordinator/src/state.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2375,6 +2375,14 @@ pub(crate) mod tests {
/// `send_prompt_core`'s `harness_mode` validation is exercisable
/// (the trait default returns `None`, which rejects every mode).
pub(crate) harness: PlMutex<Option<String>>,
/// `per_host_reserved`'s answer: the reservations tests stage on
/// the seeded hosts (empty = nothing reserved).
pub(crate) reserved:
PlMutex<std::collections::HashMap<HostId, engram_core::types::host::ReservedBudget>>,
/// Makes `per_host_reserved` fail, as a database error does.
pub(crate) reserved_read_fails: PlMutex<bool>,
/// `list_queued_sessions_fifo`'s answer.
pub(crate) queued: PlMutex<Vec<engram_core::types::session::QueuedSession>>,
}

impl MiniMeta {
Expand Down Expand Up @@ -2432,6 +2440,9 @@ pub(crate) mod tests {
ops: engram_core::types::session_op::InMemoryOpLog::default(),
outbox: PlMutex::new(Vec::new()),
harness: PlMutex::new(None),
reserved: PlMutex::new(Default::default()),
reserved_read_fails: PlMutex::new(false),
queued: PlMutex::new(Vec::new()),
}
}
}
Expand Down Expand Up @@ -2702,6 +2713,22 @@ pub(crate) mod tests {
async fn list_active_hosts(&self) -> Result<Vec<HostRecord>, MetaError> {
Ok(self.hosts.lock().clone())
}
async fn list_queued_sessions_fifo(
&self,
) -> Result<Vec<engram_core::types::session::QueuedSession>, MetaError> {
Ok(self.queued.lock().clone())
}
async fn per_host_reserved(
&self,
) -> Result<
std::collections::HashMap<HostId, engram_core::types::host::ReservedBudget>,
MetaError,
> {
if *self.reserved_read_fails.lock() {
return Err(MetaError::Db("staged read failure".into()));
}
Ok(self.reserved.lock().clone())
}
async fn set_host_status(&self, _: HostId, _: HostStatus) -> Result<(), MetaError> {
Ok(())
}
Expand Down
4 changes: 2 additions & 2 deletions crates/engram-core/src/traits/metadata.rs
Original file line number Diff line number Diff line change
Expand Up @@ -691,8 +691,8 @@ pub trait MetadataStore: Send + Sync {
/// ADR 0047/0048: per-host reserved budget (Σ `mem_budget_mib` AND
/// Σ `cpu_budget_vcpus` over the memory-reserving session states) —
/// the read-side twin of `reserve_and_persist_create`'s aggregate, for the
/// capacity-soft resume/evac picker and the fleet view. Default impl
/// (mocks): empty map (no reservations).
/// capacity-soft resume/evac picker, the fleet view and the parking
/// headroom gate. Default impl (mocks): empty map (no reservations).
async fn per_host_reserved(
&self,
) -> Result<std::collections::HashMap<HostId, crate::types::host::ReservedBudget>, MetaError>
Expand Down
105 changes: 100 additions & 5 deletions crates/engram-core/src/types/host.rs
Original file line number Diff line number Diff line change
Expand Up @@ -559,16 +559,72 @@ pub struct HostHeartbeat {
pub lease_renew_until: Option<DateTime<Utc>>,
}

/// Per-host reserved memory and CPU: Σ over the memory-reserving session
/// states of `mem_budget_mib` and `cpu_budget_vcpus`. The read-side twin
/// of `reserve_placement`'s in-transaction aggregate, for the resume/evac
/// picker and the fleet view.
#[derive(Clone, Copy, Debug, Default)]
/// Per-host reserved memory and CPU: Σ of `mem_budget_mib` and
/// `cpu_budget_vcpus` over the sessions in a memory-reserving state (a
/// parked VM included), the capturing VMs and the reserving teleport legs
/// on the host.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ReservedBudget {
pub mem_mib: i64,
pub vcpus: i64,
}

/// One host's resource accounting: what the host can hold, what the
/// coordinator has reserved on it, and what is left. Every decision that
/// asks "is there room on this host" reads it: placement (create, resume,
/// queue, capture, teleport, drain guard), parking, and the fleet's
/// autoscaling signal.
///
/// The host measures `allocatable_mib`: the RAM a reservation may use,
/// with every resident VM's memory added back, so a VM's RAM counts once,
/// in `reserved`. The coordinator owns `reserved`. Free is the
/// difference: a parked VM keeps its reservation until it descends, so
/// it is not free.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct HostHeadroom {
/// Host-measured `allocatable_mib`; `<= 0` means unmeasured (a host
/// that has not sampled its RAM yet).
pub allocatable_mib: i64,
/// `total_vcpus × overcommit` ([`host_cpu_budget`]); `0` means the
/// host has not reported its cores.
pub cpu_budget: i64,
pub reserved: ReservedBudget,
}

impl HostHeadroom {
pub fn of(host: &HostRecord, reserved: ReservedBudget) -> Self {
Self {
allocatable_mib: host.utilization.allocatable_mib as i64,
cpu_budget: host_cpu_budget(host.total_vcpus),
reserved,
}
}

/// Whether the host has measured its RAM. An unmeasured host does
/// not gate placement (a brand-new or dev host must still work) and
/// never parks a session.
pub fn measured(&self) -> bool {
self.allocatable_mib > 0
}

/// Unreserved RAM. Only meaningful when [`Self::measured`].
pub fn free_mib(&self) -> i64 {
self.allocatable_mib - self.reserved.mem_mib
}

/// Unreserved vCPU budget; `None` when the host reports no cores.
pub fn free_vcpus(&self) -> Option<i64> {
(self.cpu_budget > 0).then(|| self.cpu_budget - self.reserved.vcpus)
}

/// Whether a reservation of `mem_mib` and `vcpus` fits now. An
/// unmeasured RAM figure or an unreported core count does not gate.
pub fn fits(&self, mem_mib: i64, vcpus: i64) -> bool {
(!self.measured() || self.free_mib() >= mem_mib)
&& self.free_vcpus().is_none_or(|free| free >= vcpus)
}
}

/// ADR 0048: the CPU overcommit factor. The host CPU budget is
/// `total_vcpus × factor` — FC guests idle heavily and RAM is the hard
/// constraint, so CPU is deliberately oversubscribed to keep packing
Expand Down Expand Up @@ -950,3 +1006,42 @@ mod retirement_tests {
}
}
}

#[cfg(test)]
mod headroom_tests {
use super::{HostHeadroom, ReservedBudget};

fn room(allocatable_mib: i64, cpu_budget: i64, mem_mib: i64, vcpus: i64) -> HostHeadroom {
HostHeadroom {
allocatable_mib,
cpu_budget,
reserved: ReservedBudget { mem_mib, vcpus },
}
}

#[test]
fn free_is_allocatable_less_reserved_in_both_dimensions() {
let r = room(32_768, 32, 28_000, 30);
assert_eq!(r.free_mib(), 4_768);
assert_eq!(r.free_vcpus(), Some(2));
assert!(r.fits(4_096, 2));
assert!(!r.fits(8_192, 2), "RAM binds");
assert!(!r.fits(4_096, 4), "CPU binds");
}

#[test]
fn an_unmeasured_host_does_not_gate_placement() {
let r = room(0, 0, 0, 0);
assert!(!r.measured());
assert!(
r.fits(65_536, 64),
"unmeasured RAM and unreported cores do not gate"
);
assert_eq!(room(32_768, 0, 0, 0).free_vcpus(), None);
}

#[test]
fn an_over_reserved_host_has_negative_free_ram() {
assert_eq!(room(60_000, 0, 70_000, 0).free_mib(), -10_000);
}
}
6 changes: 4 additions & 2 deletions crates/engram-dst-cosim/src/scheduler.rs
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,9 @@ impl Cosim {
// AND `session_ops::enqueue(Evict)` — but that enqueue is the same
// detached spawn that starves the real capture I/O here, so rung 1
// applies the nomination transition directly and drives the Evict
// verb inline below (identical to what the detector enqueues).
// verb inline below. `allow_park` is false, as in a park's descent:
// the roomy simulated host would park, and the capture is what the
// callers exercise.
let _ = self
.world
.meta
Expand All @@ -236,7 +238,7 @@ impl Cosim {
self.enqueue_and_drive(
session_id,
OpKind::Evict,
serde_json::json!({ "target": "idle", "allow_park": true, "nominated": true }),
serde_json::json!({ "target": "idle", "allow_park": false, "nominated": true }),
&key,
)
.await;
Expand Down
12 changes: 8 additions & 4 deletions crates/engram-host-agent/src/ram_ledger.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,12 @@
//! sandbox's PSS is added back into `allocatable_mib` iff its session
//! holds a coordinator memory reservation. The host-side proxy for
//! "holds a reservation" is the per-sandbox `parked` flag
//! (`engram-sandbox-firecracker`'s `LiveSandbox::parked`, always
//! `false` until epic-parking-ladder lands). Parked residents are
//! (`engram-sandbox-firecracker`'s `LiveSandbox::parked`). No backend
//! sets it: a parked session keeps its reservation (the coordinator
//! counts `Parked` as a reserving state), so its VM is a reserved,
//! running resident here. A backend that sets the flag must come with a
//! coordinator that stops reserving for that session, or the VM's RAM
//! is charged twice. Parked residents are
//! charged at their full measured PSS — prod observed rss ≈ pss
//! (12.39 GB ≈ 12.39 GB) for the running dev-brain sandbox on
//! 2026-07-01, so no sharing discount is ever assumed; density math
Expand Down Expand Up @@ -65,8 +69,8 @@ pub struct RamLedgerSnapshot {
/// The only bucket ever added back into `allocatable_mib`.
pub running_vm_pss_mib: u64,
/// Σ smaps_rollup PSS over FC sandboxes flagged `parked` — RAM-
/// resident, reservation-free (epic-parking-ladder rungs 2-3).
/// Always 0 until a backend ever parks a sandbox. Never added back
/// resident, reservation-free. Always 0: no backend sets the flag
/// (see the module invariant). Never added back
/// into `allocatable_mib` — see the module invariant above. This is
/// the seam the ladder/queue-scanner later use to count
/// *reclaimable-under-pressure* capacity; deliberately NOT folded
Expand Down
2 changes: 1 addition & 1 deletion crates/engram-host-agent/src/resource.rs
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ use std::sync::Arc;
/// heartbeats report the actual machine size — without this, the
/// coord scheduler sees `total_mib=0` on every host and rejects every
/// session whose `vm_spec.memory.max_mib` is non-zero (every session
/// in practice), and the idle-evictor's `host_has_memory_headroom`
/// in practice), and the idle-evictor's park gate
/// fails closed so rung-2 park never fires (ADR 0096 — macOS/VZ hosts
/// reported 0 until this grew a Darwin arm). Returns 0 on failure —
/// the caller logs and continues; sessions just won't be scheduled to
Expand Down
Loading
Loading