diff --git a/Cargo.lock b/Cargo.lock index d6bc4cf..08cf849 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2223,7 +2223,7 @@ dependencies = [ [[package]] name = "dig-node-core" -version = "0.39.0" +version = "0.40.0" dependencies = [ "async-trait", "axum", @@ -2281,7 +2281,7 @@ dependencies = [ [[package]] name = "dig-node-service" -version = "0.90.0" +version = "0.91.0" dependencies = [ "async-trait", "axum", diff --git a/Cargo.toml b/Cargo.toml index cb52d64..5831436 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ edition = "2021" # the ROOT manifest (`[workspace.package].version`), so it MUST be set here for a # release to fire (§3.6). The library crates (dig-node-core/dig-runtime/dig-wallet) # keep their own independent versions — only the released binary tracks the workspace version. -version = "0.90.0" +version = "0.91.0" # Release hardening, matching digstore: keep integer-overflow checks ON in release. # The node parses untrusted serialized input and does offset/length arithmetic over diff --git a/crates/dig-node-core/Cargo.toml b/crates/dig-node-core/Cargo.toml index 3baec0f..714e879 100644 --- a/crates/dig-node-core/Cargo.toml +++ b/crates/dig-node-core/Cargo.toml @@ -16,7 +16,7 @@ name = "dig-node-core" # 0.31.0 adds cache observability to `cache.stats` (§7.10e) — `refetch_count` + per-tier `tiers` # occupancy fields, and makes `InboundDemand::entry_count` a real (no-longer-`#[cfg(test)]`) public # API — a new, backwards-compatible surface (dig_ecosystem#1991), hence a MINOR bump. -version = "0.39.0" +version = "0.40.0" edition = "2021" license = "GPL-2.0-only" description = "The canonical DIG node ENGINE library (crate `dig_node_core`): the JSON-RPC dispatch (`handle_rpc`, the same contract as rpc.dig.net), local-first content serve/fetch/redirect from LOCAL .dig store modules (via digstore_host::serve_blind), chain-anchored-root resolution, chain-watch + subscriptions + generation gap-fill, the LRU cache, and the full P2P stack. Shared UNCHANGED by both host shells: the `dig-node` OS-service binary (dig-node-service) and the DIG Browser's in-process cdylib (dig-runtime). Native Rust so the compiled-module serve path works." diff --git a/crates/dig-node-core/SPEC.md b/crates/dig-node-core/SPEC.md index 91bfce6..e80e44b 100644 --- a/crates/dig-node-core/SPEC.md +++ b/crates/dig-node-core/SPEC.md @@ -821,7 +821,9 @@ classified by its fields: `method` present → JSON-RPC; `length` present (no `m bring-up a standalone node with a DHT and the reshare warmer wired SPAWNS a self-driven precache loop. Each round: sample quorum-reconciled candidates from the 4a neighbourhood probe → resolve each sampled content-key to a self-verified `VerifiedCapsuleKey` (the two gates above) → score by relevance - under this node's context → select within the tier-0 sub-budget → fetch each selected store through + under this node's context (`RelevanceInputs.local_read_count` drives this tier-0 CANDIDATE selection + and is legitimately `0` for a speculative DHT-sampled candidate this node has never read) → select + within the tier-0 sub-budget → fetch each selected store through the SHARED `CapsuleWarmer` (byte-capped, chain-anchored, merkle-verified, cached tagged `Tier0Precache`, then announced as a holder). The loop is GOVERNED end to end and the following are NORMATIVE: diff --git a/crates/dig-node-core/src/inbound_demand.rs b/crates/dig-node-core/src/inbound_demand.rs index ed99ed3..6018d01 100644 --- a/crates/dig-node-core/src/inbound_demand.rs +++ b/crates/dig-node-core/src/inbound_demand.rs @@ -9,16 +9,16 @@ //! This is what THIS node fetched — gated `ReadOrigin::Local` to stay amplification-safe. //! 2. **Inbound demand (this module, #1990).** A remote PEER asks US for a resource from a store — //! direct evidence this node's neighbourhood WANTS that content. A peer's request is the demand -//! signal, so the demanded store is tagged `Tier1Demand`, its demand count feeds -//! [`relevance`](crate::relevance::relevance)'s local-demand term, and the tier gives it eviction -//! precedence over speculative `Tier0Precache`. +//! signal, so the demanded store is tagged `Tier1Demand` (via +//! [`Node::module_tier`](crate::Node)), which gives it eviction precedence over speculative +//! `Tier0Precache` — the KEEP mechanism the modules-cache sweep consults (#2013). //! //! # Why this ledger exists //! The on-disk LRU cache (see [`crate`] `DIG_NODE_CACHE_CAP`) keys entries by path and orders them by //! file mtime alone — it carries NO per-entry acquisition tier. This ledger is the FIRST live //! tier-tagging: a small, additive, in-memory map that records WHICH stores a peer has demanded and //! at what tier, WITHOUT touching the `.dig` format or the on-disk cache layout. It is the source the -//! relevance demand term + the tier-based eviction precedence consult for peer-demanded stores. +//! tier-based eviction precedence consults for peer-demanded stores. //! Process-lifetime (never persisted) — like the other §7.9 runtime counters, it resets each start. //! //! # Bounded against remote memory-exhaustion (load-bearing) diff --git a/crates/dig-node-core/src/lib.rs b/crates/dig-node-core/src/lib.rs index 208cbf9..09d5824 100644 --- a/crates/dig-node-core/src/lib.rs +++ b/crates/dig-node-core/src/lib.rs @@ -407,8 +407,9 @@ pub struct Node { chat: chat::ChatState, /// The live INBOUND-DEMAND ledger (#1990, epic #1934): the FIRST live tier-tagging. Records which /// stores a remote PEER has asked this node to serve and tags each `Tier1Demand`, so a peer's - /// request — direct evidence this node's neighbourhood wants the content — feeds the relevance - /// local-demand term and gives the store eviction precedence over speculative `Tier0Precache`. + /// request — direct evidence this node's neighbourhood wants the content — assigns the + /// `Tier1Demand` tier (via [`Node::module_tier`]) that gives the store eviction precedence over + /// speculative `Tier0Precache`. /// In-memory + process-lifetime; additive over the on-disk cache. See [`inbound_demand`]. inbound_demand: inbound_demand::InboundDemand, /// This node's own 32-byte `peer_id` (= its DHT node id — both are the SHA-256 SPKI value, one @@ -2835,36 +2836,9 @@ impl Node { &self.cache_dir } - /// The recorded inbound-demand count for a store (0 if none) — the peer-request count that feeds - /// [`RelevanceInputs::local_read_count`](crate::relevance::RelevanceInputs::local_read_count). - /// - /// This is the relevance-feed reader half of the inbound-demand ledger: the SIGNAL is recorded - /// live today (§7.10d), but the live scoring that CONSUMES it lands with the epic-#1934 cache- - /// wiring child, so the reader is currently exercised only by this crate's tests. `allow(dead_code)` - /// records that the API is deliberately ahead of its live consumer (the same shape as the pure, - /// not-yet-wired `relevance`/`tier0_selector` modules), not accidentally unused. - #[allow(dead_code)] - pub(crate) fn inbound_demand_count(&self, store_hex: &str) -> u32 { - self.inbound_demand.count(store_hex) - } - - /// The tier a store is tagged with by inbound demand, if any — always - /// [`Tier1Demand`](crate::relevance::CacheTier::Tier1Demand) when present. Like - /// [`Node::inbound_demand_count`], the reader is ahead of its live eviction-precedence consumer - /// (epic #1934 cache-wiring child) and currently exercised only by tests. - #[allow(dead_code)] - pub(crate) fn inbound_demand_tier( - &self, - store_hex: &str, - ) -> Option { - self.inbound_demand.tier(store_hex) - } - /// Distinct stores currently held in the inbound-demand ledger — the live `Tier1Demand` - /// occupancy figure `cache.stats` (#1991) reports. Real and load-bearing today (unlike - /// [`Node::inbound_demand_count`]/[`Node::inbound_demand_tier`] above, which await the - /// eviction-precedence consumer): it is the ledger's own bounded-LRU size (§7.10d), so it needs - /// no cache wiring to be an honest number. + /// occupancy figure `cache.stats` (#1991) reports. It is the ledger's own bounded-LRU size + /// (§7.10d), so it needs no cache wiring to be an honest number. pub(crate) fn inbound_demand_entry_count(&self) -> usize { self.inbound_demand.entry_count() } @@ -3853,20 +3827,20 @@ mod tests { } /// **Proves (#1990):** a peer's inbound request records demand for the store — bumping its count - /// and tagging it `Tier1Demand` — so the demand feeds relevance + eviction precedence. This is the - /// always-on, amplification-free half of the trigger (it holds no content, pulls nothing). + /// and tagging it `Tier1Demand` — so the demand assigns the tier that gives eviction precedence. + /// This is the always-on, amplification-free half of the trigger (it holds no content, pulls nothing). /// **Catches:** a demand record that fails to tag Tier1 or fails to accumulate. #[tokio::test] async fn inbound_demand_records_and_tags_tier1() { let (node, _td) = test_node(None); let store_hex = "ab".repeat(32); let root_hex = "cd".repeat(32); - assert_eq!(node.inbound_demand_count(&store_hex), 0, "undemanded → 0"); + assert_eq!(node.inbound_demand.count(&store_hex), 0, "undemanded → 0"); node.note_inbound_demand(&store_hex, &root_hex); node.note_inbound_demand(&store_hex, &root_hex); - assert_eq!(node.inbound_demand_count(&store_hex), 2, "two requests → 2"); + assert_eq!(node.inbound_demand.count(&store_hex), 2, "two requests → 2"); assert_eq!( - node.inbound_demand_tier(&store_hex), + node.inbound_demand.tier(&store_hex), Some(crate::relevance::CacheTier::Tier1Demand), "inbound demand tags Tier1Demand" ); @@ -3879,7 +3853,63 @@ mod tests { async fn inbound_demand_ignores_a_noncanonical_store() { let (node, _td) = test_node(None); node.note_inbound_demand("not-a-store", &"cd".repeat(32)); - assert_eq!(node.inbound_demand_count("not-a-store"), 0); + assert_eq!(node.inbound_demand.count("not-a-store"), 0); + } + + /// **Proves (#2013, #1990):** an inbound-DEMANDED module survives a size-cap eviction sweep that + /// sacrifices an OLDER-by-mtime tier-0 precache module — through the LIVE + /// `module_tier` → `evict_modules_locked` → `plan_module_eviction` path, not just the pure + /// `evict_key` unit test. Tier precedence (`Tier0Precache` before `Tier1Demand`) OVERRIDES recency. + /// + /// **Non-vacuous:** the demanded store A is made the OLDER file and the tier-0 store B the NEWER + /// one, so if `module_tier` were ignored (both defaulting to `Tier1Demand`, pure LRU-by-mtime) the + /// sweep would evict A and keep B — the exact OPPOSITE of what is asserted. The assertion can only + /// pass because tier beats mtime. + /// **Catches:** a regression that stops stamping the demand tier into the cache entry, or sorts + /// eviction by recency alone. + #[test] + fn inbound_demanded_module_survives_tier0_eviction_sweep() { + // A plain `#[test]` (not `#[tokio::test]`) so `ENV_GUARD` — a std `Mutex` guarding the + // process-global `DIG_NODE_CACHE`/cap config — is never held across an `.await`. + let _g = ENV_GUARD.lock().unwrap_or_else(|p| p.into_inner()); + let (node, _td) = test_node(None); + + // Isolate the config the cap is read from, then pin a tiny cap: two ~1 KiB modules exceed it, + // so the sweep must evict exactly one. + let cfg = tempfile::tempdir().unwrap(); + std::env::set_var("DIG_NODE_CACHE", cfg.path()); + let _ = std::fs::remove_file(config_path()); + set_cache_cap_bytes(1_500).unwrap(); + + // Store A: inbound-demanded → tagged `Tier1Demand`. Store B: a tier-0 precache land. + let store_a = "ab".repeat(32); + let store_b = "ba".repeat(32); + let root = "cd".repeat(32); + node.note_inbound_demand(&store_a, &root); + crate::tier0_live::mark_tier0_land(&store_b); + + let path_a = module_path(&node.cache_dir, &store_a, &root); + let path_b = module_path(&node.cache_dir, &store_b, &root); + for p in [&path_a, &path_b] { + std::fs::create_dir_all(p.parent().unwrap()).unwrap(); + std::fs::write(p, vec![0u8; 1_024]).unwrap(); + } + // A is OLDER, B is NEWER — pure LRU-by-mtime would sacrifice A, so keeping A proves tier wins. + filetime::set_file_mtime(&path_a, filetime::FileTime::from_unix_time(1_000, 0)).unwrap(); + filetime::set_file_mtime(&path_b, filetime::FileTime::from_unix_time(2_000, 0)).unwrap(); + + pin_test_rt().block_on(node.evict_modules_if_needed()); + + assert!( + path_a.exists(), + "the inbound-DEMANDED (Tier1) module must survive though it is the older file" + ); + assert!( + !path_b.exists(), + "the tier-0 precache module must be evicted first despite being the newer file" + ); + + std::env::remove_var("DIG_NODE_CACHE"); } /// **Proves (#1990):** the inbound-demand PULL is OFF by default — a peer's request records demand @@ -3906,7 +3936,7 @@ mod tests { ); let (s, r) = (store.to_hex(), tip.to_hex()); rt.block_on(async { node.note_inbound_demand(&s, &r) }); - assert_eq!(node.inbound_demand_count(&s), 1, "demand is still recorded"); + assert_eq!(node.inbound_demand.count(&s), 1, "demand is still recorded"); let key = format!("{s}:{r}"); assert!( !node.capsule_acquisition.is_warming(&key), @@ -3979,7 +4009,7 @@ mod tests { let key = format!("{s}:{r}"); let warming = node.capsule_acquisition.is_warming(&key); std::env::remove_var("DIG_NODE_INBOUND_DEMAND_CACHE"); - assert_eq!(node.inbound_demand_count(&s), 1, "demand still recorded"); + assert_eq!(node.inbound_demand.count(&s), 1, "demand still recorded"); assert!(!warming, "an already-held store claims no backfill slot"); } @@ -4029,7 +4059,7 @@ mod tests { ); rt.block_on(async { far_node.note_inbound_demand(&s, &r) }); assert_eq!( - far_node.inbound_demand_count(&s), + far_node.inbound_demand.count(&s), 1, "demand is still recorded regardless of proximity" ); diff --git a/crates/dig-node-core/src/tier0_live.rs b/crates/dig-node-core/src/tier0_live.rs index 862eced..a88c7e6 100644 --- a/crates/dig-node-core/src/tier0_live.rs +++ b/crates/dig-node-core/src/tier0_live.rs @@ -110,7 +110,7 @@ fn tier0_land_ledger() -> &'static Mutex> { } /// Record that the tier-0 loop landed `store_hex`, so the eviction sweep sacrifices it before demand. -fn mark_tier0_land(store_hex: &str) { +pub(crate) fn mark_tier0_land(store_hex: &str) { tier0_land_ledger() .lock() .unwrap_or_else(|p| p.into_inner())